/** * Reads the binary representation of any type of learner (including the label lexicon, but not * including the feature lexicon) from the given stream. The stream is expected to first return * a string containing the fully qualified class name of the learner. If the <i>short</i> value * <code>-1</code> appears instead, this method returns <code>null</code>. * * <p> * This method is appropriate for reading learners as written by * {@link #write(ExceptionlessOutputStream)}. * * @param in The input stream. * @return The learner read from the stream. **/ public static Learner readLearner(ExceptionlessInputStream in) { return readLearner(in, true); }
/** * <!-- preExtract(String,boolean) --> Performs labeled feature vector pre-extraction into the * specified file (or memory), replacing {@link #parser} with one that reads from that file (or * memory). After pre-extraction, the lexicon is written to disk. It is assumed that * {@link #learner} already knows where to write the lexicon. If it doesn't, call * {@link Learner#setLexiconLocation(String)} or * {@link Learner#setLexiconLocation(java.net.URL)} on that object before calling this method. * * @param exampleFile The full path to a file into which examples will be written, or * <code>null</code> to extract into memory. * @param zip Whether or not to compress the extracted examples. * @return The resulting lexicon. **/ public Lexicon preExtract(String exampleFile, boolean zip) { Learner preExtractLearner = preExtract(exampleFile, zip, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); return preExtractLearner.getLexicon(); }
/** * Returns the value of the discrete prediction that this learner would make, given a feature * vector. * * @param vector The example vector. * @return The discrete value. **/ public String discreteValue(FeatureVector vector) { Classifier saveExtractor = getExtractor(); Classifier saveLabeler = getLabeler(); setExtractor(new FeatureVectorReturner()); setLabeler(new LabelVectorReturner()); String result = discreteValue((Object) vector); setExtractor(saveExtractor); setLabeler(saveLabeler); return result; }
/** * Returns the value of the discrete prediction that this learner would make, given an example. * * @param example The example object. * @return The discrete value. **/ public String discreteValue(Object example) { Object[] exampleArray = getExampleArray(example, false); return discreteValue((int[]) exampleArray[0], (double[]) exampleArray[1]); }
/** * This method makes one or more decisions about a single object, returning those decisions as * {@link Feature}s in a vector. * * @param example The object to make decisions about. * @return A vector of {@link Feature}s about the input object. **/ public FeatureVector classify(Object example) { Object[] exampleArray = getExampleArray(example, false); return classify((int[]) exampleArray[0], (double[]) exampleArray[1]); }
/** * Returns the value of the real prediction that this learner would make, given an example. * * @param example The example object. * @return The real value. **/ public double realValue(Object example) { Object[] exampleArray = getExampleArray(example, false); return realValue((int[]) exampleArray[0], (double[]) exampleArray[1]); }
/** * Reads the binary representation of a learner with this object's run-time type, overwriting * any and all learned or manually specified parameters as well as the label lexicon but without * modifying the feature lexicon. * * @param in The input stream. **/ public void read(ExceptionlessInputStream in) { super.read(in); baseLTU = (LinearThresholdUnit) Learner.readLearner(in); conjunctiveLabels = in.readBoolean(); int N = in.readInt(); network = new OVector(N); for (int i = 0; i < N; ++i) network.add(Learner.readLearner(in)); }
public static void main(String[] args) { String exFileName = null; String lexFileName = null; String lcFileName = null; try { exFileName = args[0]; lexFileName = args[1]; lcFileName = args[2]; if (args.length > 3) throw new Exception(); } catch (Exception e) { System.err .println("usage: java edu.illinois.cs.cogcomp.lbjava.parse.ArrayFileParser <example file> <lexicon file> <lc file>"); System.exit(1); } ArrayFileParser parser = new ArrayFileParser(exFileName); Learner learner = Learner.readLearner(lcFileName); learner.readLexicon(lexFileName); for (Object e = parser.next(); e != null; e = parser.next()) { FeatureVector v = new FeatureVector((Object[]) e, learner.getLexicon(), learner.getLabelLexicon()); v.sort(); System.out.println(v); } } }
/** * Writes the learned function's internal representation in binary form. * * @param out The output stream. **/ public void write(ExceptionlessOutputStream out) { super.write(out); write(out, table); }
/** * Reads the binary representation of any type of learner (including the label lexicon, but not * including the feature lexicon), with the option of cutting off the reading process after the * label lexicon and before any learned parameters. When <code>whole</code> is * <code>false</code>, the reading process is cut off in this way. * * <p> * This method is appropriate for reading learners as written by * {@link #write(ExceptionlessOutputStream)}. * * @param in The input stream. * @param whole Whether or not to read the whole model. * @return The learner read from the stream. **/ public static Learner readLearner(ExceptionlessInputStream in, boolean whole) { String name = in.readString(); if (name == null) return null; Learner result = ClassUtils.getLearner(name); result.unclone(); if (whole) result.read(in); // Overridden by decendents else { result.readLabelLexicon(in); // Should not be overridden by decendents Lexicon labelLexicon = result.getLabelLexicon(); result.forget(); result.setLabelLexicon(labelLexicon); } return result; }
Learner preExtractLearner = learner.emptyClone(); preExtractLearner.setLabelLexicon(learner.getLabelLexicon()); Lexicon lexicon = learner.getLexicon(); preExtractLearner.setLexicon(lexicon); preExtractLearner.countFeatures(countPolicy); learner.setLexicon(null); setIsTraining(true); examples = 0; Object[] exampleArray = alreadyExtracted ? (Object[]) example : preExtractLearner .getExampleArray(example); lexiconSize = preExtractLearner.getLexicon().size(); parser = new ArrayFileParser(baos.toByteArray(), zip); learner.setLabelLexicon(preExtractLearner.getLabelLexicon()); return preExtractLearner;
learner.beginTraining(); try { if (lce.parser != null) { System.gc(); } else learner.saveLexicon(); int trainingRounds = 1; Learner.writeParameters(bestParameters, parametersPath); System.out.println(" " + getName() + ": Training on entire training set"); learner.saveLexicon(); // Writes .lex even if lexicon is empty. } finally { learner.doneTraining(); new Accuracy(true).test(learner, learner.getLabeler(), testParser); learner.save(); // Doesn't write .lex if lexicon is empty. } catch (Exception e) { System.err.println("LBJava ERROR: Exception while training " + getName() + ":");
continue; String prediction = candidates[i].discreteValue(t); preBIOLevel1[i] = prediction; if (prediction.startsWith("B") || prediction.startsWith("U")){ ScoreSet scores = candidates[i].scores(t); Score[] scoresArray = scores.toArray(); for (Score s : scoresArray){ return new Pair<>(candidates[chosen].discreteValue(t), chosen);
/** * Produces a set of scores indicating the degree to which each possible discrete classification * value is associated with the given feature vector. Learners that return a <code>real</code> * feature or more than one feature may implement this method by simply returning * <code>null</code>. * * @param vector The vector to make decisions about. * @return A set of scores indicating the degree to which each possible discrete classification * value is associated with the given example vector. **/ public ScoreSet scores(FeatureVector vector) { Classifier saveExtractor = getExtractor(); Classifier saveLabeler = getLabeler(); setExtractor(new FeatureVectorReturner()); setLabeler(new LabelVectorReturner()); ScoreSet result = scores((Object) vector); setExtractor(saveExtractor); setLabeler(saveLabeler); return result; }
/** * Reads the binary representation of a learner with this object's run-time type, overwriting * any and all learned or manually specified parameters as well as the label lexicon but without * modifying the feature lexicon. * * @param in The input stream. **/ public void read(ExceptionlessInputStream in) { super.read(in); forget(); read(in, table); }
/** * This method makes one or more decisions about a single feature vector, returning those * decisions as {@link Feature}s in a vector. * * @param vector The vector to make decisions about. * @return A vector of {@link Feature}s about the input vector. **/ public FeatureVector classify(FeatureVector vector) { Classifier saveExtractor = getExtractor(); Classifier saveLabeler = getLabeler(); setExtractor(new FeatureVectorReturner()); setLabeler(new LabelVectorReturner()); FeatureVector result = classify((Object) vector); setExtractor(saveExtractor); setLabeler(saveLabeler); return result; }
/** * Returns the value of the real prediction that this learner would make, given a feature * vector. * * @param vector The example vector. * @return The real value. **/ public double realValue(FeatureVector vector) { Classifier saveExtractor = getExtractor(); Classifier saveLabeler = getLabeler(); setExtractor(new FeatureVectorReturner()); setLabeler(new LabelVectorReturner()); double result = realValue((Object) vector); setExtractor(saveExtractor); setLabeler(saveLabeler); return result; }
public TestReal(Learner classifier, Classifier oracle, Parser parser) { int examples = 0; double totalDifference = 0; double[] actuals = {}; double[] predictions = {}; classifier.write(System.out); for (Object example = parser.next(); example != null; example = parser.next()) { double prediction = classifier.realValue(example); predictions = Arrays.copyOf(predictions, predictions.length + 1); predictions[predictions.length - 1] = prediction; double value = oracle.realValue(example); actuals = Arrays.copyOf(actuals, actuals.length + 1); actuals[actuals.length - 1] = value; double difference = Math.abs(prediction - value); totalDifference += difference; classifier.classify(example); ++examples; System.out.println("Example " + examples + " difference: " + difference + " (prediction: " + prediction + ")"); } System.out.println("test examples number: " + examples); double avg = totalDifference / examples; System.out.println("Average difference: " + avg); double p = getPearsonCorrelation(predictions, actuals); System.out.println("Pearson correlation:" + p); SpearmansCorrelation e = new SpearmansCorrelation(); double sp = e.correlation(predictions, actuals); System.out.println("Spearman correlation:" + sp); }
lexicon = trainer.preExtract(exFilePath, preExtractZip); else if (lce.pruneStatus != RevisionAnalysis.UNAFFECTED) lexicon = learner.getLexiconDiscardCounts(); else trainer.fillInSizes(); else learner.getLexicon().perClassToGlobalCounts(); : !pruningPolicy.isNone()) { trainer.pruneDataset(exFilePath, preExtractZip, pruningPolicy, preExtractLearner); lexicon = preExtractLearner.getLexicon(); if (preExtractLearner == learner) learner.setLexicon(null); .exists()))) { if (lexicon == null) learner.readLexiconOnDemand(classDir + getName() + ".lex"); else { learner.setLexicon(lexicon); HashSet dependors = (HashSet) SemanticAnalysis.dependorGraph.get(name); if (lexicon != null && dependors.size() > 0) learner.setLexicon(lexicon); else learner.readLexiconOnDemand(classDir + name + ".lex");
/** * Resets the internal bookkeeping. **/ public void forget() { super.forget(); numClasses = numFeatures = 0; allLabels = null; allExamples = null; weights = null; conjunctiveLabels = false; }