@Override protected void setup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); String priorClustersPath = conf.get(ClusterIterator.PRIOR_PATH_KEY); classifier = new ClusterClassifier(); classifier.readFromSeqFiles(conf, new Path(priorClustersPath)); policy = classifier.getPolicy(); policy.update(classifier); super.setup(context); }
@Override protected void setup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); String priorClustersPath = conf.get(ClusterIterator.PRIOR_PATH_KEY); classifier = new ClusterClassifier(); classifier.readFromSeqFiles(conf, new Path(priorClustersPath)); policy = classifier.getPolicy(); policy.update(classifier); super.setup(context); }
@Override protected void reduce(IntWritable key, Iterable<ClusterWritable> values, Context context) throws IOException, InterruptedException { Iterator<ClusterWritable> iter = values.iterator(); Cluster first = iter.next().getValue(); // there must always be at least one while (iter.hasNext()) { Cluster cluster = iter.next().getValue(); first.observe(cluster); } List<Cluster> models = Lists.newArrayList(); models.add(first); classifier = new ClusterClassifier(models, policy); classifier.close(); context.write(key, new ClusterWritable(first)); }
@Override protected void setup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); String priorClustersPath = conf.get(ClusterIterator.PRIOR_PATH_KEY); classifier = new ClusterClassifier(); classifier.readFromSeqFiles(conf, new Path(priorClustersPath)); policy = classifier.getPolicy(); policy.update(classifier); super.setup(context); }
@Override protected void reduce(IntWritable key, Iterable<ClusterWritable> values, Context context) throws IOException, InterruptedException { Iterator<ClusterWritable> iter = values.iterator(); Cluster first = iter.next().getValue(); // there must always be at least one while (iter.hasNext()) { Cluster cluster = iter.next().getValue(); first.observe(cluster); } List<Cluster> models = Lists.newArrayList(); models.add(first); classifier = new ClusterClassifier(models, policy); classifier.close(); context.write(key, new ClusterWritable(first)); }
@Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); Configuration conf = context.getConfiguration(); String clustersIn = conf.get(ClusterClassificationConfigKeys.CLUSTERS_IN); threshold = conf.getFloat(ClusterClassificationConfigKeys.OUTLIER_REMOVAL_THRESHOLD, 0.0f); emitMostLikely = conf.getBoolean(ClusterClassificationConfigKeys.EMIT_MOST_LIKELY, false); clusterModels = Lists.newArrayList(); if (clustersIn != null && !clustersIn.isEmpty()) { Path clustersInPath = new Path(clustersIn); clusterModels = populateClusterModels(clustersInPath, conf); ClusteringPolicy policy = ClusterClassifier .readPolicy(finalClustersPath(clustersInPath)); clusterClassifier = new ClusterClassifier(clusterModels, policy); } clusterId = new IntWritable(); }
@Override protected void reduce(IntWritable key, Iterable<ClusterWritable> values, Context context) throws IOException, InterruptedException { Iterator<ClusterWritable> iter = values.iterator(); Cluster first = iter.next().getValue(); // there must always be at least one while (iter.hasNext()) { Cluster cluster = iter.next().getValue(); first.observe(cluster); } List<Cluster> models = new ArrayList<>(); models.add(first); classifier = new ClusterClassifier(models, policy); classifier.close(); context.write(key, new ClusterWritable(first)); }
@Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); Configuration conf = context.getConfiguration(); String clustersIn = conf.get(ClusterClassificationConfigKeys.CLUSTERS_IN); threshold = conf.getFloat(ClusterClassificationConfigKeys.OUTLIER_REMOVAL_THRESHOLD, 0.0f); emitMostLikely = conf.getBoolean(ClusterClassificationConfigKeys.EMIT_MOST_LIKELY, false); clusterModels = new ArrayList<>(); if (clustersIn != null && !clustersIn.isEmpty()) { Path clustersInPath = new Path(clustersIn); clusterModels = populateClusterModels(clustersInPath, conf); ClusteringPolicy policy = ClusterClassifier .readPolicy(finalClustersPath(clustersInPath)); clusterClassifier = new ClusterClassifier(clusterModels, policy); } clusterId = new IntWritable(); }
@Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); Configuration conf = context.getConfiguration(); String clustersIn = conf.get(ClusterClassificationConfigKeys.CLUSTERS_IN); threshold = conf.getFloat(ClusterClassificationConfigKeys.OUTLIER_REMOVAL_THRESHOLD, 0.0f); emitMostLikely = conf.getBoolean(ClusterClassificationConfigKeys.EMIT_MOST_LIKELY, false); clusterModels = Lists.newArrayList(); if (clustersIn != null && !clustersIn.isEmpty()) { Path clustersInPath = new Path(clustersIn); clusterModels = populateClusterModels(clustersInPath, conf); ClusteringPolicy policy = ClusterClassifier .readPolicy(finalClustersPath(clustersInPath)); clusterClassifier = new ClusterClassifier(clusterModels, policy); } clusterId = new IntWritable(); }
@Override protected void setup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); String priorClustersPath = conf.get(ClusterIterator.PRIOR_PATH_KEY); classifier = new ClusterClassifier(); classifier.readFromSeqFiles(conf, new Path(priorClustersPath)); policy = classifier.getPolicy(); policy.update(classifier); super.setup(context); }
@Override protected void setup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); String priorClustersPath = conf.get(ClusterIterator.PRIOR_PATH_KEY); classifier = new ClusterClassifier(); classifier.readFromSeqFiles(conf, new Path(priorClustersPath)); policy = classifier.getPolicy(); policy.update(classifier); super.setup(context); }
@Override protected void setup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); String priorClustersPath = conf.get(ClusterIterator.PRIOR_PATH_KEY); classifier = new ClusterClassifier(); classifier.readFromSeqFiles(conf, new Path(priorClustersPath)); policy = classifier.getPolicy(); policy.update(classifier); super.setup(context); }
private static void classifyClusterSeq(Configuration conf, Path input, Path clusters, Path output, Double clusterClassificationThreshold, boolean emitMostLikely) throws IOException { List<Cluster> clusterModels = populateClusterModels(clusters, conf); ClusteringPolicy policy = ClusterClassifier.readPolicy(finalClustersPath(conf, clusters)); ClusterClassifier clusterClassifier = new ClusterClassifier(clusterModels, policy); selectCluster(input, clusterModels, clusterClassifier, output, clusterClassificationThreshold, emitMostLikely); }
private static void classifyClusterSeq(Configuration conf, Path input, Path clusters, Path output, Double clusterClassificationThreshold, boolean emitMostLikely) throws IOException { List<Cluster> clusterModels = populateClusterModels(clusters, conf); ClusteringPolicy policy = ClusterClassifier.readPolicy(finalClustersPath(conf, clusters)); ClusterClassifier clusterClassifier = new ClusterClassifier(clusterModels, policy); selectCluster(input, clusterModels, clusterClassifier, output, clusterClassificationThreshold, emitMostLikely); }
private static void classifyClusterSeq(Configuration conf, Path input, Path clusters, Path output, Double clusterClassificationThreshold, boolean emitMostLikely) throws IOException { List<Cluster> clusterModels = populateClusterModels(clusters, conf); ClusteringPolicy policy = ClusterClassifier.readPolicy(finalClustersPath(conf, clusters)); ClusterClassifier clusterClassifier = new ClusterClassifier(clusterModels, policy); selectCluster(input, clusterModels, clusterClassifier, output, clusterClassificationThreshold, emitMostLikely); }
private ClusterClassifier writeAndRead(ClusterClassifier classifier) throws IOException { Path path = new Path(getTestTempDirPath(), "output"); classifier.writeToSeqFiles(path); ClusterClassifier newClassifier = new ClusterClassifier(); newClassifier.readFromSeqFiles(getConfiguration(), path); return newClassifier; }
private static ClusterClassifier newSoftClusterClassifier() { List<Cluster> models = Lists.newArrayList(); DistanceMeasure measure = new ManhattanDistanceMeasure(); models.add(new SoftCluster(new DenseVector(2).assign(1), 0, measure)); models.add(new SoftCluster(new DenseVector(2), 1, measure)); models.add(new SoftCluster(new DenseVector(2).assign(-1), 2, measure)); return new ClusterClassifier(models, new FuzzyKMeansClusteringPolicy()); }
private static ClusterClassifier newDMClassifier() { List<Cluster> models = Lists.newArrayList(); DistanceMeasure measure = new ManhattanDistanceMeasure(); models.add(new DistanceMeasureCluster(new DenseVector(2).assign(1), 0, measure)); models.add(new DistanceMeasureCluster(new DenseVector(2), 1, measure)); models.add(new DistanceMeasureCluster(new DenseVector(2).assign(-1), 2, measure)); return new ClusterClassifier(models, new KMeansClusteringPolicy()); }
private static ClusterClassifier newKlusterClassifier() { List<Cluster> models = Lists.newArrayList(); DistanceMeasure measure = new ManhattanDistanceMeasure(); models.add(new org.apache.mahout.clustering.kmeans.Kluster(new DenseVector(2).assign(1), 0, measure)); models.add(new org.apache.mahout.clustering.kmeans.Kluster(new DenseVector(2), 1, measure)); models.add(new org.apache.mahout.clustering.kmeans.Kluster(new DenseVector(2).assign(-1), 2, measure)); return new ClusterClassifier(models, new KMeansClusteringPolicy()); }
private static ClusterClassifier newCosineKlusterClassifier() { List<Cluster> models = Lists.newArrayList(); DistanceMeasure measure = new CosineDistanceMeasure(); models.add(new org.apache.mahout.clustering.kmeans.Kluster(new DenseVector(2).assign(1), 0, measure)); models.add(new org.apache.mahout.clustering.kmeans.Kluster(new DenseVector(2), 1, measure)); models.add(new org.apache.mahout.clustering.kmeans.Kluster(new DenseVector(2).assign(-1), 2, measure)); return new ClusterClassifier(models, new KMeansClusteringPolicy()); }