WeightedPropertyVectorWritable value = new WeightedPropertyVectorWritable();
@Override public String toString() { Vector vector = getVector(); StringBuilder bldr = new StringBuilder("wt: ").append(getWeight()).append(' '); if (properties != null && !properties.isEmpty()) { for (Map.Entry<Text, Text> entry : properties.entrySet()) { bldr.append(entry.getKey().toString()).append(": ").append(entry.getValue().toString()).append(' '); } } bldr.append(" vec: ").append(vector == null ? "null" : AbstractCluster.formatVector(vector, null)); return bldr.toString(); }
private static void write(List<Cluster> clusterModels, SequenceFile.Writer writer, WeightedPropertyVectorWritable weightedPropertyVectorWritable, int maxValueIndex) throws IOException { Cluster cluster = clusterModels.get(maxValueIndex); DistanceMeasureCluster distanceMeasureCluster = (DistanceMeasureCluster) cluster; DistanceMeasure distanceMeasure = distanceMeasureCluster.getMeasure(); double distance = distanceMeasure.distance(cluster.getCenter(), weightedPropertyVectorWritable.getVector()); weightedPropertyVectorWritable.getProperties().put(new Text("distance"), new Text(Double.toString(distance))); writer.append(new IntWritable(cluster.getId()), weightedPropertyVectorWritable); }
private void collectVectorsForAssertion() throws IOException { Path[] partFilePaths = FileUtil.stat2Paths(fs .globStatus(classifiedOutputPath)); FileStatus[] listStatus = fs.listStatus(partFilePaths, PathFilters.partFilter()); for (FileStatus partFile : listStatus) { SequenceFile.Reader classifiedVectors = new SequenceFile.Reader(fs, partFile.getPath(), conf); Writable clusterIdAsKey = new IntWritable(); WeightedPropertyVectorWritable point = new WeightedPropertyVectorWritable(); while (classifiedVectors.next(clusterIdAsKey, point)) { collectVector(clusterIdAsKey.toString(), point.getVector()); } } }
WeightedPropertyVectorWritable value = new WeightedPropertyVectorWritable(); while (reader.next(key, value)) { System.out.println(value.toString() + " belongs to cluster " + key.toString());
List<WeightedPropertyVectorWritable> wpvList = collector.getValue(k); assertTrue("empty cluster!", !wpvList.isEmpty()); if (wpvList.get(0).getVector().get(0) <= 2.0) { for (WeightedPropertyVectorWritable wv : wpvList) { Vector v = wv.getVector(); int idx = v.maxValueIndex(); assertTrue("bad cluster!", v.get(idx) <= 2.0); } else { for (WeightedPropertyVectorWritable wv : wpvList) { Vector v = wv.getVector(); int idx = v.minValueIndex(); assertTrue("bad cluster!", v.get(idx) > 2.0);
private void write(VectorWritable vw, Context context, int clusterIndex, double weight) throws IOException, InterruptedException { Cluster cluster = clusterModels.get(clusterIndex); clusterId.set(cluster.getId()); DistanceMeasureCluster distanceMeasureCluster = (DistanceMeasureCluster) cluster; DistanceMeasure distanceMeasure = distanceMeasureCluster.getMeasure(); double distance = distanceMeasure.distance(cluster.getCenter(), vw.get()); Map<Text, Text> props = Maps.newHashMap(); props.put(new Text("distance"), new Text(Double.toString(distance))); context.write(clusterId, new WeightedPropertyVectorWritable(weight, vw.get(), props)); }
@Override public String toString() { Vector vector = getVector(); StringBuilder bldr = new StringBuilder("wt: ").append(getWeight()).append(' '); if (properties != null && !properties.isEmpty()) { for (Map.Entry<Text, Text> entry : properties.entrySet()) { bldr.append(entry.getKey().toString()).append(": ").append(entry.getValue().toString()).append(' '); } } bldr.append(" vec: ").append(vector == null ? "null" : AbstractCluster.formatVector(vector, null)); return bldr.toString(); }
private static void write(List<Cluster> clusterModels, SequenceFile.Writer writer, WeightedPropertyVectorWritable weightedPropertyVectorWritable, int maxValueIndex) throws IOException { Cluster cluster = clusterModels.get(maxValueIndex); DistanceMeasureCluster distanceMeasureCluster = (DistanceMeasureCluster) cluster; DistanceMeasure distanceMeasure = distanceMeasureCluster.getMeasure(); double distance = distanceMeasure.distance(cluster.getCenter(), weightedPropertyVectorWritable.getVector()); weightedPropertyVectorWritable.getProperties().put(new Text("distance"), new Text(Double.toString(distance))); writer.append(new IntWritable(cluster.getId()), weightedPropertyVectorWritable); }
private void write(VectorWritable vw, Context context, int clusterIndex, double weight) throws IOException, InterruptedException { Cluster cluster = clusterModels.get(clusterIndex); clusterId.set(cluster.getId()); DistanceMeasureCluster distanceMeasureCluster = (DistanceMeasureCluster) cluster; DistanceMeasure distanceMeasure = distanceMeasureCluster.getMeasure(); double distance = distanceMeasure.distance(cluster.getCenter(), vw.get()); Map<Text, Text> props = Maps.newHashMap(); props.put(new Text("distance"), new Text(Double.toString(distance))); context.write(clusterId, new WeightedPropertyVectorWritable(weight, vw.get(), props)); }
@Override public String toString() { Vector vector = getVector(); StringBuilder bldr = new StringBuilder("wt: ").append(getWeight()).append(' '); if (properties != null && !properties.isEmpty()) { for (Map.Entry<Text, Text> entry : properties.entrySet()) { bldr.append(entry.getKey().toString()).append(": ").append(entry.getValue().toString()).append(' '); } } bldr.append(" vec: ").append(vector == null ? "null" : AbstractCluster.formatVector(vector, null)); return bldr.toString(); }
private static void write(List<Cluster> clusterModels, SequenceFile.Writer writer, WeightedPropertyVectorWritable weightedPropertyVectorWritable, int maxValueIndex) throws IOException { Cluster cluster = clusterModels.get(maxValueIndex); DistanceMeasureCluster distanceMeasureCluster = (DistanceMeasureCluster) cluster; DistanceMeasure distanceMeasure = distanceMeasureCluster.getMeasure(); double distance = distanceMeasure.distance(cluster.getCenter(), weightedPropertyVectorWritable.getVector()); weightedPropertyVectorWritable.getProperties().put(new Text("distance"), new Text(Double.toString(distance))); writer.append(new IntWritable(cluster.getId()), weightedPropertyVectorWritable); }
private void write(VectorWritable vw, Context context, int clusterIndex, double weight) throws IOException, InterruptedException { Cluster cluster = clusterModels.get(clusterIndex); clusterId.set(cluster.getId()); DistanceMeasureCluster distanceMeasureCluster = (DistanceMeasureCluster) cluster; DistanceMeasure distanceMeasure = distanceMeasureCluster.getMeasure(); double distance = distanceMeasure.distance(cluster.getCenter(), vw.get()); Map<Text, Text> props = new HashMap<>(); props.put(new Text("distance"), new Text(Double.toString(distance))); context.write(clusterId, new WeightedPropertyVectorWritable(weight, vw.get(), props)); }
private static void classifyAndWrite(List<Cluster> clusterModels, Double clusterClassificationThreshold, boolean emitMostLikely, SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException { Map<Text, Text> props = Maps.newHashMap(); if (emitMostLikely) { int maxValueIndex = pdfPerCluster.maxValueIndex(); WeightedPropertyVectorWritable weightedPropertyVectorWritable = new WeightedPropertyVectorWritable(pdfPerCluster.maxValue(), vw.get(), props); write(clusterModels, writer, weightedPropertyVectorWritable, maxValueIndex); } else { writeAllAboveThreshold(clusterModels, clusterClassificationThreshold, writer, vw, pdfPerCluster); } }
private static void classifyAndWrite(List<Cluster> clusterModels, Double clusterClassificationThreshold, boolean emitMostLikely, SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException { Map<Text, Text> props = new HashMap<>(); if (emitMostLikely) { int maxValueIndex = pdfPerCluster.maxValueIndex(); WeightedPropertyVectorWritable weightedPropertyVectorWritable = new WeightedPropertyVectorWritable(pdfPerCluster.maxValue(), vw.get(), props); write(clusterModels, writer, weightedPropertyVectorWritable, maxValueIndex); } else { writeAllAboveThreshold(clusterModels, clusterClassificationThreshold, writer, vw, pdfPerCluster); } }
private static void classifyAndWrite(List<Cluster> clusterModels, Double clusterClassificationThreshold, boolean emitMostLikely, SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException { Map<Text, Text> props = Maps.newHashMap(); if (emitMostLikely) { int maxValueIndex = pdfPerCluster.maxValueIndex(); WeightedPropertyVectorWritable weightedPropertyVectorWritable = new WeightedPropertyVectorWritable(pdfPerCluster.maxValue(), vw.get(), props); write(clusterModels, writer, weightedPropertyVectorWritable, maxValueIndex); } else { writeAllAboveThreshold(clusterModels, clusterClassificationThreshold, writer, vw, pdfPerCluster); } }
private static void writeAllAboveThreshold(List<Cluster> clusterModels, Double clusterClassificationThreshold, SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException { Map<Text, Text> props = Maps.newHashMap(); for (Element pdf : pdfPerCluster.nonZeroes()) { if (pdf.get() >= clusterClassificationThreshold) { WeightedPropertyVectorWritable wvw = new WeightedPropertyVectorWritable(pdf.get(), vw.get(), props); int clusterIndex = pdf.index(); write(clusterModels, writer, wvw, clusterIndex); } } }
private static void writeAllAboveThreshold(List<Cluster> clusterModels, Double clusterClassificationThreshold, SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException { Map<Text, Text> props = Maps.newHashMap(); for (Element pdf : pdfPerCluster.nonZeroes()) { if (pdf.get() >= clusterClassificationThreshold) { WeightedPropertyVectorWritable wvw = new WeightedPropertyVectorWritable(pdf.get(), vw.get(), props); int clusterIndex = pdf.index(); write(clusterModels, writer, wvw, clusterIndex); } } }
private static void writeAllAboveThreshold(List<Cluster> clusterModels, Double clusterClassificationThreshold, SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException { Map<Text, Text> props = new HashMap<>(); for (Element pdf : pdfPerCluster.nonZeroes()) { if (pdf.get() >= clusterClassificationThreshold) { WeightedPropertyVectorWritable wvw = new WeightedPropertyVectorWritable(pdf.get(), vw.get(), props); int clusterIndex = pdf.index(); write(clusterModels, writer, wvw, clusterIndex); } } }