/** * Returns the next {@link LinkedVector} of {@link Token}s. * * @return The next {@link LinkedVector} of {@link Token}s parsed, or * <code>null</code> if there are no more children in the stream. **/ public Object next() { return convert((LinkedVector) parser.next()); }
/** * Given a {@link LinkedVector} containing {@link Word}s, this method * creates a new {@link LinkedVector} containing {@link Token}s. * * @param v A {@link LinkedVector} of {@link Word}s. * @return A {@link LinkedVector} of {@link Token}s corresponding to the * input {@link Word}s. **/ public static LinkedVector convert(LinkedVector v) { if (v == null) return null; if (v.size() == 0) return v; Word w = (Word) v.get(0); Token t = new Token(w, null, null); for (w = (Word) w.next; w != null; w = (Word) w.next) { t.next = new Token(w, t, null); t = (Token) t.next; } return new LinkedVector(t); }
public static void chunkTester(String testFile){ Parser parser; parser = new CoNLL2000Parser(testFile); BIOTester tester = new BIOTester(new Chunker(), new ChunkLabel(), new ChildrenFromVectors(parser)); tester.test().printPerformance(System.out); } public static void chunkTester(String testFile, String modeldir, String modelname){
/** * Implements the program described above. * * @param args The command line parameters. **/ public static void main(String[] args) { // Parse the command line // if (args.length != 1) { // logger.error("usage: java edu.illinois.cs.cogcomp.lbj.pos.TestPOS <text file>"); // System.exit(1); // } // String testingFile = args[0]; ResourceManager rm = new POSConfigurator().getDefaultConfig(); String testingFile = rm.getString("testData"); TestDiscrete.testDiscrete(new TestDiscrete(), new POSTagger(), new POSLabel(), new POSBracketToToken(testingFile), true, 0); } }
/** * Given a {@link LinkedVector} containing {@link Word}s, this method * creates a new {@link LinkedVector} containing {@link Token}s. * * @param v A {@link LinkedVector} of {@link Word}s. * @return A {@link LinkedVector} of {@link Token}s corresponding to the * input {@link Word}s. **/ public static LinkedVector convert(LinkedVector v) { if (v == null) return null; if (v.size() == 0) return v; Word w = (Word) v.get(0); Token t = new Token(w, null, null); for (w = (Word) w.next; w != null; w = (Word) w.next) { t.next = new Token(w, t, null); t = (Token) t.next; } return new LinkedVector(t); }
public static void chunkTester(String testFile, String modeldir, String modelname){ Parser parser; String lcpath =modeldir+ File.separator+modelname+".lc"; String lexpath = modeldir+File.separator+modelname+".lex"; parser = new CoNLL2000Parser(testFile); BIOTester tester = new BIOTester(new Chunker(lcpath,lexpath), new ChunkLabel(), new ChildrenFromVectors(parser)); tester.test().printPerformance(System.out); } public static void main(String[] args){
/** * Returns the next {@link LinkedVector} of {@link Token}s. * * @return The next {@link LinkedVector} of {@link Token}s parsed, or * <code>null</code> if there are no more children in the stream. **/ public Object next() { return convert((LinkedVector) parser.next()); }
/** * Implements the program described above. * * @param args The command line parameters. **/ public static void main(String[] args) { // Parse the command line // if (args.length != 1) { // logger.error("usage: java edu.illinois.cs.cogcomp.lbj.pos.TestPOS <text file>"); // System.exit(1); // } // String testingFile = args[0]; ResourceManager rm = new POSConfigurator().getDefaultConfig(); String testingFile = rm.getString("testData"); TestDiscrete.testDiscrete(new TestDiscrete(), new POSTagger(), new POSLabel(), new POSBracketToToken(testingFile), true, 0); } }
public IllinoisPOSHandler() { super("Illinois Part-Of-Speech Tagger", "0.2", "illinoispos"); logger.info("Loading POS model.."); tagger.discreteValue(new Token(new Word("The"), null, "")); logger.info("POS Tagger ready"); tokensfield = CuratorViewNames.tokens; sentencesfield = CuratorViewNames.sentences; }
public static void chunkTester(String testFile, String modeldir, String modelname){ Parser parser; String lcpath =modeldir+ File.separator+modelname+".lc"; String lexpath = modeldir+File.separator+modelname+".lex"; parser = new CoNLL2000Parser(testFile); BIOTester tester = new BIOTester(new Chunker(lcpath,lexpath), new ChunkLabel(), new ChildrenFromVectors(parser)); tester.test().printPerformance(System.out); } public static void main(String[] args){
/** * This method returns {@link Token}s until the input is exhausted, at * which point it returns <code>null</code>. **/ public Object next() { while (next == null) { LinkedVector words = (LinkedVector) parser.next(); if (words == null) return null; Word w = (Word) words.get(0); Token t = new Token(w, null, null); for (w = (Word) w.next; w != null; w = (Word) w.next) { t.next = new Token(w, t, null); t = (Token) t.next; } LinkedVector tokens = new LinkedVector(t); next = (Token) tokens.get(0); } Token result = next; next = (Token) next.next; return result; }
public static void chunkTester(String testFile){ Parser parser; parser = new CoNLL2000Parser(testFile); BIOTester tester = new BIOTester(new Chunker(), new ChunkLabel(), new ChildrenFromVectors(parser)); tester.test().printPerformance(System.out); } public static void chunkTester(String testFile, String modeldir, String modelname){
/** * This method returns {@link Token}s until the input is exhausted, at * which point it returns <code>null</code>. **/ public Object next() { while (next == null) { LinkedVector words = (LinkedVector) parser.next(); if (words == null) return null; Word w = (Word) words.get(0); Token t = new Token(w, null, null); for (w = (Word) w.next; w != null; w = (Word) w.next) { t.next = new Token(w, t, null); t = (Token) t.next; } LinkedVector tokens = new LinkedVector(t); next = (Token) tokens.get(0); } Token result = next; next = (Token) next.next; return result; }
/** * Returns the next labeled word in the data. */ public Object next() { if (currentWord == null) { LinkedVector vector = (LinkedVector) super.next(); while (vector != null && vector.size() == 0) vector = (LinkedVector) super.next(); if (vector == null) return null; Word w = (Word) vector.get(0); Token t = currentWord = new Token(w, null, w.partOfSpeech); t.partOfSpeech = null; while (w.next != null) { w = (Word) w.next; t.next = new Token(w, t, w.partOfSpeech); t.partOfSpeech = null; t = (Token) t.next; } } Token result = currentWord; currentWord = (Token) currentWord.next; return result; } }
/** * Returns the next labeled word in the data. */ public Object next() { if (currentWord == null) { LinkedVector vector = (LinkedVector) super.next(); while (vector != null && vector.size() == 0) vector = (LinkedVector) super.next(); if (vector == null) return null; Word w = (Word) vector.get(0); Token t = currentWord = new Token(w, null, w.partOfSpeech); t.partOfSpeech = null; while (w.next != null) { w = (Word) w.next; t.next = new Token(w, t, w.partOfSpeech); t.partOfSpeech = null; t = (Token) t.next; } } Token result = currentWord; currentWord = (Token) currentWord.next; return result; } }
/** * Produces the next object parsed from the input file; in this case, that object is guaranteed * to be a <code>LinkedVector</code> populated by <code>Token</code>s representing a sentence. **/ public Object next() { String[] line = (String[]) super.next(); while (line != null && (line.length < 2 || line[4].equals("-X-"))) line = (String[]) super.next(); if (line == null) return null; if (line[3].charAt(0) == 'I') line[3] = "B" + line[3].substring(1); Token t = new Token(new Word(line[5], line[4]), null, line[3]); String previous = line[3]; for (line = (String[]) super.next(); line != null && line.length > 0; line = (String[]) super.next()) { if (line[3].charAt(0) == 'I' && !previous.endsWith(line[3].substring(2))) line[3] = "B" + line[3].substring(1); t.next = new Token(new Word(line[5], line[4]), t, line[3]); t = (Token) t.next; previous = line[3]; } return new LinkedVector(t); } }
/** * Produces the next object parsed from the input file; in this case, that object is guaranteed * to be a <code>LinkedVector</code> populated by <code>Token</code>s representing a sentence. **/ public Object next() { String[] line = (String[]) super.next(); while (line != null && (line.length < 2 || line[4].equals("-X-"))) line = (String[]) super.next(); if (line == null) return null; if (line[3].charAt(0) == 'I') line[3] = "B" + line[3].substring(1); Token t = new Token(new Word(line[5], line[4]), null, line[3]); String previous = line[3]; for (line = (String[]) super.next(); line != null && line.length > 0; line = (String[]) super.next()) { if (line[3].charAt(0) == 'I' && !previous.endsWith(line[3].substring(2))) line[3] = "B" + line[3].substring(1); t.next = new Token(new Word(line[5], line[4]), t, line[3]); t = (Token) t.next; previous = line[3]; } return new LinkedVector(t); } }
/** * Produces the next object parsed from the input file; in this case, that object is guaranteed * to be a <code>LinkedVector</code> populated by <code>Token</code>s representing a sentence. **/ public Object next() { String[] line = (String[]) super.next(); while (line != null && line.length == 0) line = (String[]) super.next(); if (line == null) return null; String pos = line[1]; if (pos.equals("-")) pos = null; Token t = new Token(new Word(line[0], pos), null, line[2]); for (line = (String[]) super.next(); line != null && line.length > 0; line = (String[]) super.next()) { pos = line[1]; if (pos.equals("-")) pos = null; t.next = new Token(new Word(line[0], pos), t, line[2]); t = (Token) t.next; } return new LinkedVector(t); } }