WeightedPropertyVectorWritable value = new WeightedPropertyVectorWritable();
private void write(VectorWritable vw, Context context, int clusterIndex, double weight) throws IOException, InterruptedException { Cluster cluster = clusterModels.get(clusterIndex); clusterId.set(cluster.getId()); DistanceMeasureCluster distanceMeasureCluster = (DistanceMeasureCluster) cluster; DistanceMeasure distanceMeasure = distanceMeasureCluster.getMeasure(); double distance = distanceMeasure.distance(cluster.getCenter(), vw.get()); Map<Text, Text> props = Maps.newHashMap(); props.put(new Text("distance"), new Text(Double.toString(distance))); context.write(clusterId, new WeightedPropertyVectorWritable(weight, vw.get(), props)); }
private void write(VectorWritable vw, Context context, int clusterIndex, double weight) throws IOException, InterruptedException { Cluster cluster = clusterModels.get(clusterIndex); clusterId.set(cluster.getId()); DistanceMeasureCluster distanceMeasureCluster = (DistanceMeasureCluster) cluster; DistanceMeasure distanceMeasure = distanceMeasureCluster.getMeasure(); double distance = distanceMeasure.distance(cluster.getCenter(), vw.get()); Map<Text, Text> props = Maps.newHashMap(); props.put(new Text("distance"), new Text(Double.toString(distance))); context.write(clusterId, new WeightedPropertyVectorWritable(weight, vw.get(), props)); }
private void write(VectorWritable vw, Context context, int clusterIndex, double weight) throws IOException, InterruptedException { Cluster cluster = clusterModels.get(clusterIndex); clusterId.set(cluster.getId()); DistanceMeasureCluster distanceMeasureCluster = (DistanceMeasureCluster) cluster; DistanceMeasure distanceMeasure = distanceMeasureCluster.getMeasure(); double distance = distanceMeasure.distance(cluster.getCenter(), vw.get()); Map<Text, Text> props = new HashMap<>(); props.put(new Text("distance"), new Text(Double.toString(distance))); context.write(clusterId, new WeightedPropertyVectorWritable(weight, vw.get(), props)); }
private static void classifyAndWrite(List<Cluster> clusterModels, Double clusterClassificationThreshold, boolean emitMostLikely, SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException { Map<Text, Text> props = Maps.newHashMap(); if (emitMostLikely) { int maxValueIndex = pdfPerCluster.maxValueIndex(); WeightedPropertyVectorWritable weightedPropertyVectorWritable = new WeightedPropertyVectorWritable(pdfPerCluster.maxValue(), vw.get(), props); write(clusterModels, writer, weightedPropertyVectorWritable, maxValueIndex); } else { writeAllAboveThreshold(clusterModels, clusterClassificationThreshold, writer, vw, pdfPerCluster); } }
private static void classifyAndWrite(List<Cluster> clusterModels, Double clusterClassificationThreshold, boolean emitMostLikely, SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException { Map<Text, Text> props = new HashMap<>(); if (emitMostLikely) { int maxValueIndex = pdfPerCluster.maxValueIndex(); WeightedPropertyVectorWritable weightedPropertyVectorWritable = new WeightedPropertyVectorWritable(pdfPerCluster.maxValue(), vw.get(), props); write(clusterModels, writer, weightedPropertyVectorWritable, maxValueIndex); } else { writeAllAboveThreshold(clusterModels, clusterClassificationThreshold, writer, vw, pdfPerCluster); } }
private static void classifyAndWrite(List<Cluster> clusterModels, Double clusterClassificationThreshold, boolean emitMostLikely, SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException { Map<Text, Text> props = Maps.newHashMap(); if (emitMostLikely) { int maxValueIndex = pdfPerCluster.maxValueIndex(); WeightedPropertyVectorWritable weightedPropertyVectorWritable = new WeightedPropertyVectorWritable(pdfPerCluster.maxValue(), vw.get(), props); write(clusterModels, writer, weightedPropertyVectorWritable, maxValueIndex); } else { writeAllAboveThreshold(clusterModels, clusterClassificationThreshold, writer, vw, pdfPerCluster); } }
private static void writeAllAboveThreshold(List<Cluster> clusterModels, Double clusterClassificationThreshold, SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException { Map<Text, Text> props = Maps.newHashMap(); for (Element pdf : pdfPerCluster.nonZeroes()) { if (pdf.get() >= clusterClassificationThreshold) { WeightedPropertyVectorWritable wvw = new WeightedPropertyVectorWritable(pdf.get(), vw.get(), props); int clusterIndex = pdf.index(); write(clusterModels, writer, wvw, clusterIndex); } } }
private static void writeAllAboveThreshold(List<Cluster> clusterModels, Double clusterClassificationThreshold, SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException { Map<Text, Text> props = Maps.newHashMap(); for (Element pdf : pdfPerCluster.nonZeroes()) { if (pdf.get() >= clusterClassificationThreshold) { WeightedPropertyVectorWritable wvw = new WeightedPropertyVectorWritable(pdf.get(), vw.get(), props); int clusterIndex = pdf.index(); write(clusterModels, writer, wvw, clusterIndex); } } }
private static void writeAllAboveThreshold(List<Cluster> clusterModels, Double clusterClassificationThreshold, SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException { Map<Text, Text> props = new HashMap<>(); for (Element pdf : pdfPerCluster.nonZeroes()) { if (pdf.get() >= clusterClassificationThreshold) { WeightedPropertyVectorWritable wvw = new WeightedPropertyVectorWritable(pdf.get(), vw.get(), props); int clusterIndex = pdf.index(); write(clusterModels, writer, wvw, clusterIndex); } } }
WeightedPropertyVectorWritable value = new WeightedPropertyVectorWritable(); while (reader.next(key, value)) { System.out.println(value.toString() + " belongs to cluster " + key.toString());
private void collectVectorsForAssertion() throws IOException { Path[] partFilePaths = FileUtil.stat2Paths(fs .globStatus(classifiedOutputPath)); FileStatus[] listStatus = fs.listStatus(partFilePaths, PathFilters.partFilter()); for (FileStatus partFile : listStatus) { SequenceFile.Reader classifiedVectors = new SequenceFile.Reader(fs, partFile.getPath(), conf); Writable clusterIdAsKey = new IntWritable(); WeightedPropertyVectorWritable point = new WeightedPropertyVectorWritable(); while (classifiedVectors.next(clusterIdAsKey, point)) { collectVector(clusterIdAsKey.toString(), point.getVector()); } } }