/** * <!-- preExtract(String,boolean) --> Performs labeled feature vector pre-extraction into the * specified file (or memory), replacing {@link #parser} with one that reads from that file (or * memory). After pre-extraction, the lexicon is written to disk. It is assumed that * {@link #learner} already knows where to write the lexicon. If it doesn't, call * {@link Learner#setLexiconLocation(String)} or * {@link Learner#setLexiconLocation(java.net.URL)} on that object before calling this method. * * @param exampleFile The full path to a file into which examples will be written, or * <code>null</code> to extract into memory. * @param zip Whether or not to compress the extracted examples. * @return The resulting lexicon. **/ public Lexicon preExtract(String exampleFile, boolean zip) { Learner preExtractLearner = preExtract(exampleFile, zip, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); return preExtractLearner.getLexicon(); }
else learner.getLexicon().perClassToGlobalCounts(); : !pruningPolicy.isNone()) { trainer.pruneDataset(exFilePath, preExtractZip, pruningPolicy, preExtractLearner); lexicon = preExtractLearner.getLexicon(); if (preExtractLearner == learner) learner.setLexicon(null);
Lexicon lexicon = preExtractLearner.getLexicon();
public static void main(String[] args) { String exFileName = null; String lexFileName = null; String lcFileName = null; try { exFileName = args[0]; lexFileName = args[1]; lcFileName = args[2]; if (args.length > 3) throw new Exception(); } catch (Exception e) { System.err .println("usage: java edu.illinois.cs.cogcomp.lbjava.parse.ArrayFileParser <example file> <lexicon file> <lc file>"); System.exit(1); } ArrayFileParser parser = new ArrayFileParser(exFileName); Learner learner = Learner.readLearner(lcFileName); learner.readLexicon(lexFileName); for (Object e = parser.next(); e != null; e = parser.next()) { FeatureVector v = new FeatureVector((Object[]) e, learner.getLexicon(), learner.getLabelLexicon()); v.sort(); System.out.println(v); } } }
Learner preExtractLearner = learner.emptyClone(); preExtractLearner.setLabelLexicon(learner.getLabelLexicon()); Lexicon lexicon = learner.getLexicon(); preExtractLearner.setLexicon(lexicon); preExtractLearner.countFeatures(countPolicy); lexiconSize = preExtractLearner.getLexicon().size();
Learner preExtractLearner = trainer.preExtract("models/relation_classifier_all", true, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); Lexicon lexicon = preExtractLearner.getLexicon(); classifier.setLexicon(lexicon); int examples = train_parser.relations_bi.size(); classifier.initialize(examples, preExtractLearner.getLexicon().size()); for (Relation r : train_parser.relations_bi){ classifier.learn(r);
Learner preExtractLearner = trainer.preExtract("models/SEMEVAL.ex", true, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); Lexicon lexicon = preExtractLearner.getLexicon(); classifier.setLexicon(lexicon); int examples = 0; classifier.initialize(examples, preExtractLearner.getLexicon().size()); for (Object example = train_parser.next(); example != null; example = train_parser.next()){ classifier.learn(example);
Learner preExtractLearner = trainer.preExtract("models/relation_classifier_fold_" + i + ".ex", true, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); Lexicon lexicon = preExtractLearner.getLexicon(); classifier.setLexicon(lexicon); int examples = train_parser.relations_bi.size(); classifier.initialize(examples, preExtractLearner.getLexicon().size()); for (Relation r : train_parser.relations_bi){ classifier.learn(r);
public static void generateModel(String serializedDataInput, String modelLoc){ ACEMentionReader train_parser = IOHelper.serializeDataIn(serializedDataInput); relation_classifier classifier = new relation_classifier(); classifier.setLexiconLocation(modelLoc + ".lex"); BatchTrainer trainer = new BatchTrainer(classifier, train_parser); Learner preExtractLearner = trainer.preExtract(modelLoc + ".ex", true, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); Lexicon lexicon = preExtractLearner.getLexicon(); classifier.setLexicon(lexicon); int examples = train_parser.relations_bi.size(); classifier.initialize(examples, preExtractLearner.getLexicon().size()); for (Relation r : train_parser.relations_bi){ classifier.learn(r); } classifier.doneWithRound(); classifier.doneLearning(); classifier.setModelLocation(modelLoc + ".lc"); classifier.saveModel(); }
Learner preExtractLearner = trainer.preExtract(modelFileName + ".ex", true, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); Lexicon lexicon = preExtractLearner.getLexicon(); classifier.setLexicon(lexicon); int examples = 0; classifier.initialize(examples, preExtractLearner.getLexicon().size()); for (Object example = train_parser.next(); example != null; example = train_parser.next()){ classifier.learn(example);
Learner preExtractLearner = trainer.preExtract(modelFileName + ".ex", true, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); Lexicon lexicon = preExtractLearner.getLexicon(); classifier.setLexicon(lexicon); int examples = 0; classifier.initialize(examples, preExtractLearner.getLexicon().size()); for (Object example = train_parser.next(); example != null; example = train_parser.next()){ classifier.learn(example);
Learner preExtractLearner = trainer.preExtract(modelFileName + ".ex", true, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); Lexicon lexicon = preExtractLearner.getLexicon(); classifier.setLexicon(lexicon); int examples = 0; classifier.initialize(examples, preExtractLearner.getLexicon().size()); for (Object example = train_parser.next(); example != null; example = train_parser.next()){ classifier.learn(example);
Learner preExtractLearner = trainer.preExtract(modelFileName + ".ex", true, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); Lexicon lexicon = preExtractLearner.getLexicon(); classifier.setLexicon(lexicon); int examples = 0; classifier.initialize(examples, preExtractLearner.getLexicon().size()); for (int i = 0; i < 1; i++) { train_parser.reset();
Learner preExtractLearner = trainer.preExtract(modelFileName + ".ex", true, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); Lexicon lexicon = preExtractLearner.getLexicon(); classifier.setLexicon(lexicon); int examples = 0; classifier.initialize(examples, preExtractLearner.getLexicon().size()); for (Object example = train_parser.next(); example != null; example = train_parser.next()){ classifier.learn(example);
Learner preExtractLearner = trainer.preExtract(modelFileName + ".ex", true, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); Lexicon lexicon = preExtractLearner.getLexicon(); classifier.setLexicon(lexicon); int examples = 0; classifier.initialize(examples, preExtractLearner.getLexicon().size()); for (int i = 0; i < 1; i++) { train_parser.reset();
Learner preExtractLearner = trainer.preExtract(modelFileName + ".ex", true, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); Lexicon lexicon = preExtractLearner.getLexicon(); classifier.setLexicon(lexicon); int examples = 0; classifier.initialize(examples, preExtractLearner.getLexicon().size()); for (Object example = train_parser.next(); example != null; example = train_parser.next()){ classifier.learn(example);
Learner preExtractLearner = trainer.preExtract(modelFileName + ".ex", true, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); Lexicon lexicon = preExtractLearner.getLexicon(); classifier.setLexicon(lexicon); int examples = 0; classifier.initialize(examples, preExtractLearner.getLexicon().size()); for (Object example = train_parser.next(); example != null; example = train_parser.next()){ classifier.learn(example);
Learner preExtractLearner = trainer.preExtract(modelFileName + ".ex", true, Lexicon.CountPolicy.none); preExtractLearner.saveLexicon(); Lexicon lexicon = preExtractLearner.getLexicon(); classifier.setLexicon(lexicon); int examples = 0; classifier.initialize(examples, preExtractLearner.getLexicon().size()); for (int i = 0; i < 1; i++) { train_parser.reset();