/** * Computes the summaries for the distances in each cluster. * @param datapoints iterable of datapoints. * @param centroids iterable of Centroids. * @return a list of OnlineSummarizers where the i-th element is the summarizer corresponding to the cluster whose * index is i. */ public static List<OnlineSummarizer> summarizeClusterDistances(Iterable<? extends Vector> datapoints, Iterable<? extends Vector> centroids, DistanceMeasure distanceMeasure) { UpdatableSearcher searcher = new ProjectionSearch(distanceMeasure, 3, 1); searcher.addAll(centroids); List<OnlineSummarizer> summarizers = new ArrayList<>(); if (searcher.size() == 0) { return summarizers; } for (int i = 0; i < searcher.size(); ++i) { summarizers.add(new OnlineSummarizer()); } for (Vector v : datapoints) { Centroid closest = (Centroid)searcher.search(v, 1).get(0).getValue(); OnlineSummarizer summarizer = summarizers.get(closest.getIndex()); summarizer.add(distanceMeasure.distance(v, closest)); } return summarizers; }
@Test public void testSearchFirst() { searcher.clear(); searcher.addAll(dataPoints); for (Vector datapoint : dataPoints) { WeightedThing<Vector> first = searcher.searchFirst(datapoint, false); WeightedThing<Vector> second = searcher.searchFirst(datapoint, true); List<WeightedThing<Vector>> firstTwo = searcher.search(datapoint, 2); assertEquals("First isn't self", 0, first.getWeight(), 0); assertEquals("First isn't self", datapoint, first.getValue()); assertEquals("First doesn't match", first, firstTwo.get(0)); assertEquals("Second doesn't match", second, firstTwo.get(1)); } }
/** * Computes the summaries for the distances in each cluster. * @param datapoints iterable of datapoints. * @param centroids iterable of Centroids. * @return a list of OnlineSummarizers where the i-th element is the summarizer corresponding to the cluster whose * index is i. */ public static List<OnlineSummarizer> summarizeClusterDistances(Iterable<? extends Vector> datapoints, Iterable<? extends Vector> centroids, DistanceMeasure distanceMeasure) { UpdatableSearcher searcher = new ProjectionSearch(distanceMeasure, 3, 1); searcher.addAll(centroids); List<OnlineSummarizer> summarizers = Lists.newArrayList(); if (searcher.size() == 0) { return summarizers; } for (int i = 0; i < searcher.size(); ++i) { summarizers.add(new OnlineSummarizer()); } for (Vector v : datapoints) { Centroid closest = (Centroid)searcher.search(v, 1).get(0).getValue(); OnlineSummarizer summarizer = summarizers.get(closest.getIndex()); summarizer.add(distanceMeasure.distance(v, closest)); } return summarizers; }
@Test public void testRemove() { searcher.clear(); for (int i = 0; i < dataPoints.rowSize(); ++i) { Vector datapoint = dataPoints.viewRow(i); searcher.add(datapoint); // As long as points are not searched for right after being added, in FastProjectionSearch, points are not // merged with the main list right away, so if a search for a point occurs before it's merged the pendingAdditions // list also needs to be looked at. // This used to not be the case for searchFirst(), thereby causing removal failures. if (i % 2 == 0) { assertTrue("Failed to find self [search]", searcher.search(datapoint, 1).get(0).getWeight() < Constants.EPSILON); assertTrue("Failed to find self [searchFirst]", searcher.searchFirst(datapoint, false).getWeight() < Constants.EPSILON); assertTrue("Failed to remove self", searcher.remove(datapoint, Constants.EPSILON)); } } } }
/** * Computes the summaries for the distances in each cluster. * @param datapoints iterable of datapoints. * @param centroids iterable of Centroids. * @return a list of OnlineSummarizers where the i-th element is the summarizer corresponding to the cluster whose * index is i. */ public static List<OnlineSummarizer> summarizeClusterDistances(Iterable<? extends Vector> datapoints, Iterable<? extends Vector> centroids, DistanceMeasure distanceMeasure) { UpdatableSearcher searcher = new ProjectionSearch(distanceMeasure, 3, 1); searcher.addAll(centroids); List<OnlineSummarizer> summarizers = Lists.newArrayList(); if (searcher.size() == 0) { return summarizers; } for (int i = 0; i < searcher.size(); ++i) { summarizers.add(new OnlineSummarizer()); } for (Vector v : datapoints) { Centroid closest = (Centroid)searcher.search(v, 1).get(0).getValue(); OnlineSummarizer summarizer = summarizers.get(closest.getIndex()); summarizer.add(distanceMeasure.distance(v, closest)); } return summarizers; }
@Test public void testSearchLimiting() { searcher.clear(); searcher.addAll(dataPoints); for (Vector datapoint : dataPoints) { List<WeightedThing<Vector>> firstTwo = searcher.search(datapoint, 2); assertThat("Search limit isn't respected", firstTwo.size(), is(lessThanOrEqualTo(2))); } }
row.assign(Functions.mult(1 / scale)); WeightedThing<Vector> cluster = clustering.search(row, 1).get(0); current.assign(cluster.getValue()); current.assign(Functions.mult(scale));
int size0 = searcher.size(); List<WeightedThing<Vector>> r0 = searcher.search(x.get(0), 2); List<WeightedThing<Vector>> r = searcher.search(x.get(0), 1); assertTrue("Vector should be gone", r.get(0).getWeight() > 0); assertEquals("Previous second neighbor should be first", 0, assertEquals(size0 - 2, searcher.size()); r = searcher.search(x.get(1), 1); assertTrue("Vector should be gone", r.get(0).getWeight() > 0);
WeightedThing<Vector> v = searcher.search(mean, 1).get(0); maxWeight = Math.max(v.getWeight(), maxWeight);
@Test public void testNearMatch() { searcher.clear(); List<MatrixSlice> queries = Lists.newArrayList(Iterables.limit(dataPoints, 100)); searcher.addAllMatrixSlicesAsWeightedVectors(dataPoints); MultiNormal noise = new MultiNormal(0.01, new DenseVector(20)); for (MatrixSlice slice : queries) { Vector query = slice.vector(); final Vector epsilon = noise.sample(); List<WeightedThing<Vector>> r = searcher.search(query, 2); query = query.plus(epsilon); assertEquals("Distance has to be small", epsilon.norm(2), r.get(0).getWeight(), 1.0e-1); assertEquals("Answer must be substantially the same as query", epsilon.norm(2), r.get(0).getValue().minus(query).norm(2), 1.0e-1); assertTrue("Wrong answer must be further away", r.get(1).getWeight() > r.get(0).getWeight()); } }
@Test public void testExactMatch() { searcher.clear(); Iterable<MatrixSlice> data = dataPoints; final Iterable<MatrixSlice> batch1 = Iterables.limit(data, 300); List<MatrixSlice> queries = Lists.newArrayList(Iterables.limit(batch1, 100)); // adding the data in multiple batches triggers special code in some searchers searcher.addAllMatrixSlices(batch1); assertEquals(300, searcher.size()); Vector q = Iterables.get(data, 0).vector(); List<WeightedThing<Vector>> r = searcher.search(q, 2); assertEquals(0, r.get(0).getValue().minus(q).norm(1), 1.0e-8); final Iterable<MatrixSlice> batch2 = Iterables.limit(Iterables.skip(data, 300), 10); searcher.addAllMatrixSlices(batch2); assertEquals(310, searcher.size()); q = Iterables.get(data, 302).vector(); r = searcher.search(q, 2); assertEquals(0, r.get(0).getValue().minus(q).norm(1), 1.0e-8); searcher.addAllMatrixSlices(Iterables.skip(data, 310)); assertEquals(dataPoints.numRows(), searcher.size()); for (MatrixSlice query : queries) { r = searcher.search(query.vector(), 2); assertEquals("Distance has to be about zero", 0, r.get(0).getWeight(), 1.0e-6); assertEquals("Answer must be substantially the same as query", 0, r.get(0).getValue().minus(query.vector()).norm(1), 1.0e-8); assertTrue("Wrong answer must have non-zero distance", r.get(1).getWeight() > r.get(0).getWeight()); } }
WeightedThing<Vector> v = searcher.search(mean, 1).get(0); summarizer.add(v.getWeight());
@Test public void testOrdering() { searcher.clear(); Matrix queries = new DenseMatrix(100, 20); MultiNormal gen = new MultiNormal(20); for (int i = 0; i < 100; i++) { queries.viewRow(i).assign(gen.sample()); } searcher.addAllMatrixSlices(dataPoints); for (MatrixSlice query : queries) { List<WeightedThing<Vector>> r = searcher.search(query.vector(), 200); double x = 0; for (WeightedThing<Vector> thing : r) { assertTrue("Scores must be monotonic increasing", thing.getWeight() >= x); x = thing.getWeight(); } } }