public IllinoisPOSHandler() { super("Illinois Part-Of-Speech Tagger", "0.2", "illinoispos"); logger.info("Loading POS model.."); tagger.discreteValue(new Token(new Word("The"), null, "")); logger.info("POS Tagger ready"); tokensfield = CuratorViewNames.tokens; sentencesfield = CuratorViewNames.sentences; }
/** * Given a {@link LinkedVector} containing {@link Word}s, this method * creates a new {@link LinkedVector} containing {@link Token}s. * * @param v A {@link LinkedVector} of {@link Word}s. * @return A {@link LinkedVector} of {@link Token}s corresponding to the * input {@link Word}s. **/ public static LinkedVector convert(LinkedVector v) { if (v == null) return null; if (v.size() == 0) return v; Word w = (Word) v.get(0); Token t = new Token(w, null, null); for (w = (Word) w.next; w != null; w = (Word) w.next) { t.next = new Token(w, t, null); t = (Token) t.next; } return new LinkedVector(t); }
/** * Given a {@link LinkedVector} containing {@link Word}s, this method * creates a new {@link LinkedVector} containing {@link Token}s. * * @param v A {@link LinkedVector} of {@link Word}s. * @return A {@link LinkedVector} of {@link Token}s corresponding to the * input {@link Word}s. **/ public static LinkedVector convert(LinkedVector v) { if (v == null) return null; if (v.size() == 0) return v; Word w = (Word) v.get(0); Token t = new Token(w, null, null); for (w = (Word) w.next; w != null; w = (Word) w.next) { t.next = new Token(w, t, null); t = (Token) t.next; } return new LinkedVector(t); }
public IllinoisChunkerHandler(String configFilename) { super("Illinois Chunker", "0.3", "illinoischunker"); logger.info("Loading Chunker model.."); tagger.discreteValue(new Token(new Word("The"), null, "")); logger.info("Chunker ready"); // XXX If no configuration file is give use the default values from CuratorViewNames if (configFilename.trim().equals("")) { tokensfield = CuratorViewNames.tokens; sentencesfield = CuratorViewNames.sentences; posfield = CuratorViewNames.pos; } else { Properties config = new Properties(); try { FileInputStream in = new FileInputStream(configFilename); config.load(new BufferedInputStream(in)); in.close(); } catch (IOException e) { logger.warn("Error reading configuration file. {}", configFilename); } tokensfield = config.getProperty("tokens.field", CuratorViewNames.tokens ); sentencesfield = config.getProperty("sentences.field", CuratorViewNames.sentences ); posfield = config.getProperty("pos.field", CuratorViewNames.pos ); } }
/** * This method returns {@link Token}s until the input is exhausted, at * which point it returns <code>null</code>. **/ public Object next() { while (next == null) { LinkedVector words = (LinkedVector) parser.next(); if (words == null) return null; Word w = (Word) words.get(0); Token t = new Token(w, null, null); for (w = (Word) w.next; w != null; w = (Word) w.next) { t.next = new Token(w, t, null); t = (Token) t.next; } LinkedVector tokens = new LinkedVector(t); next = (Token) tokens.get(0); } Token result = next; next = (Token) next.next; return result; }
/** * This method returns {@link Token}s until the input is exhausted, at * which point it returns <code>null</code>. **/ public Object next() { while (next == null) { LinkedVector words = (LinkedVector) parser.next(); if (words == null) return null; Word w = (Word) words.get(0); Token t = new Token(w, null, null); for (w = (Word) w.next; w != null; w = (Word) w.next) { t.next = new Token(w, t, null); t = (Token) t.next; } LinkedVector tokens = new LinkedVector(t); next = (Token) tokens.get(0); } Token result = next; next = (Token) next.next; return result; }
/** * Returns the next labeled word in the data. */ public Object next() { if (currentWord == null) { LinkedVector vector = (LinkedVector) super.next(); while (vector != null && vector.size() == 0) vector = (LinkedVector) super.next(); if (vector == null) return null; Word w = (Word) vector.get(0); Token t = currentWord = new Token(w, null, w.partOfSpeech); t.partOfSpeech = null; while (w.next != null) { w = (Word) w.next; t.next = new Token(w, t, w.partOfSpeech); t.partOfSpeech = null; t = (Token) t.next; } } Token result = currentWord; currentWord = (Token) currentWord.next; return result; } }
/** * Returns the next labeled word in the data. */ public Object next() { if (currentWord == null) { LinkedVector vector = (LinkedVector) super.next(); while (vector != null && vector.size() == 0) vector = (LinkedVector) super.next(); if (vector == null) return null; Word w = (Word) vector.get(0); Token t = currentWord = new Token(w, null, w.partOfSpeech); t.partOfSpeech = null; while (w.next != null) { w = (Word) w.next; t.next = new Token(w, t, w.partOfSpeech); t.partOfSpeech = null; t = (Token) t.next; } } Token result = currentWord; currentWord = (Token) currentWord.next; return result; } }
/** * Produces the next object parsed from the input file; in this case, that object is guaranteed * to be a <code>LinkedVector</code> populated by <code>Token</code>s representing a sentence. **/ public Object next() { String[] line = (String[]) super.next(); while (line != null && (line.length < 2 || line[4].equals("-X-"))) line = (String[]) super.next(); if (line == null) return null; if (line[3].charAt(0) == 'I') line[3] = "B" + line[3].substring(1); Token t = new Token(new Word(line[5], line[4]), null, line[3]); String previous = line[3]; for (line = (String[]) super.next(); line != null && line.length > 0; line = (String[]) super.next()) { if (line[3].charAt(0) == 'I' && !previous.endsWith(line[3].substring(2))) line[3] = "B" + line[3].substring(1); t.next = new Token(new Word(line[5], line[4]), t, line[3]); t = (Token) t.next; previous = line[3]; } return new LinkedVector(t); } }
/** * Produces the next object parsed from the input file; in this case, that object is guaranteed * to be a <code>LinkedVector</code> populated by <code>Token</code>s representing a sentence. **/ public Object next() { String[] line = (String[]) super.next(); while (line != null && (line.length < 2 || line[4].equals("-X-"))) line = (String[]) super.next(); if (line == null) return null; if (line[3].charAt(0) == 'I') line[3] = "B" + line[3].substring(1); Token t = new Token(new Word(line[5], line[4]), null, line[3]); String previous = line[3]; for (line = (String[]) super.next(); line != null && line.length > 0; line = (String[]) super.next()) { if (line[3].charAt(0) == 'I' && !previous.endsWith(line[3].substring(2))) line[3] = "B" + line[3].substring(1); t.next = new Token(new Word(line[5], line[4]), t, line[3]); t = (Token) t.next; previous = line[3]; } return new LinkedVector(t); } }
Token tcurrent = new Token(wcurrent, tprevious, ""); lbjTokens.add(tcurrent); if (tprevious != null) {
Token tcurrent = new Token(wcurrent, tprevious, ""); lbjTokens.add(tcurrent); if (tprevious != null) {
/** * Produces the next object parsed from the input file; in this case, that object is guaranteed * to be a <code>LinkedVector</code> populated by <code>Token</code>s representing a sentence. **/ public Object next() { String[] line = (String[]) super.next(); while (line != null && line.length == 0) line = (String[]) super.next(); if (line == null) return null; String pos = line[1]; if (pos.equals("-")) pos = null; Token t = new Token(new Word(line[0], pos), null, line[2]); for (line = (String[]) super.next(); line != null && line.length > 0; line = (String[]) super.next()) { pos = line[1]; if (pos.equals("-")) pos = null; t.next = new Token(new Word(line[0], pos), t, line[2]); t = (Token) t.next; } return new LinkedVector(t); } }
Token tcurrent = new Token(wcurrent, tprevious, ""); lbjTokens.add(tcurrent); if (tprevious != null) {
/** * Produces the next object parsed from the input file; in this case, that object is guaranteed * to be a <code>LinkedVector</code> populated by <code>Token</code>s representing a sentence. **/ public Object next() { String[] line = (String[]) super.next(); while (line != null && line.length == 0) line = (String[]) super.next(); if (line == null) return null; String pos = line[1]; if (pos.equals("-")) pos = null; Token t = new Token(new Word(line[0], pos), null, line[2]); for (line = (String[]) super.next(); line != null && line.length > 0; line = (String[]) super.next()) { pos = line[1]; if (pos.equals("-")) pos = null; t.next = new Token(new Word(line[0], pos), t, line[2]); t = (Token) t.next; } return new LinkedVector(t); } }
Token tcurrent = new Token(wcurrent, tprevious, ""); lbjTokens.add(tcurrent); if (tprevious != null) {
Token tcurrent = new Token(wcurrent, tprevious, ""); lbjSentenceTokens.add(tcurrent); if (tprevious != null) {