public TokenSequence (Token[] tokens) { this (tokens.length); for (int i = 0; i < tokens.length; i++) this.add( tokens[i] ); }
public TokenSequence (Token[] tokens) { this (tokens.length); for (int i = 0; i < tokens.length; i++) this.add( tokens[i] ); }
public TokenSequence (Token[] tokens) { this (tokens.length); for (int i = 0; i < tokens.length; i++) this.add( tokens[i] ); }
public TokenSequence (Object[] tokens) { this( tokens.length ); for (int i = 0; i < tokens.length; i++) this.add (new Token( tokens[i].toString())); }
private TokenSequence makeTokenSeq(List<String> vals) { TokenSequence ts = new TokenSequence(vals.size()); for (String s : vals) { ts.add(s); } return ts; } }
public TokenSequence (Object[] tokens) { this( tokens.length ); for (int i = 0; i < tokens.length; i++) this.add (new Token( tokens[i].toString())); }
public TokenSequence (Object[] tokens) { this( tokens.length ); for (int i = 0; i < tokens.length; i++) this.add (new Token( tokens[i].toString())); }
private TokenSequence makeTokenSeq(List<String> vals) { TokenSequence ts = new TokenSequence(vals.size()); for (String s : vals) { ts.add(s); } return ts; } }
private TokenSequence makeTokenSeq(List<String> vals) { TokenSequence ts = new TokenSequence(vals.size()); for (String s : vals) { ts.add(s); } return ts; } }
public TokenSequence randomTokenSequence (Randoms r, int length) { FeatureSequence fs = randomFeatureSequence (r, length); TokenSequence ts = new TokenSequence (length); for (int i = 0; i < length; i++) ts.add (fs.getObjectAtPosition(i).toString()); return ts; }
public TokenSequence randomTokenSequence (Randoms r, int length) { FeatureSequence fs = randomFeatureSequence (r, length); TokenSequence ts = new TokenSequence (length); for (int i = 0; i < length; i++) ts.add (fs.getObjectAtPosition(i).toString()); return ts; }
public TokenSequence randomTokenSequence (Randoms r, int length) { FeatureSequence fs = randomFeatureSequence (r, length); TokenSequence ts = new TokenSequence (length); for (int i = 0; i < length; i++) ts.add (fs.getObjectAtPosition(i).toString()); return ts; }
public Instance pipe (Instance carrier) { CharSequence string = (CharSequence) carrier.getData(); lexer.setCharSequence (string); TokenSequence ts = new StringTokenization (string); while (lexer.hasNext()) { lexer.next(); ts.add (new StringSpan (string, lexer.getStartOffset (), lexer.getEndOffset ())); } carrier.setData(ts); return carrier; }
public Instance pipe (Instance carrier) { CharSequence string = (CharSequence) carrier.getData(); lexer.setCharSequence (string); TokenSequence ts = new StringTokenization (string); while (lexer.hasNext()) { lexer.next(); ts.add (new StringSpan (string, lexer.getStartOffset (), lexer.getEndOffset ())); } carrier.setData(ts); return carrier; }
public Instance pipe (Instance carrier) { CharSequence string = (CharSequence) carrier.getData(); lexer.setCharSequence (string); TokenSequence ts = new StringTokenization (string); while (lexer.hasNext()) { lexer.next(); ts.add (new StringSpan (string, lexer.getStartOffset (), lexer.getEndOffset ())); } carrier.setData(ts); return carrier; }
public TextInstance(TextSentence textSentence, Alphabet targetAlphabet) { super(new TokenSequence(), new LabelSequence(targetAlphabet), textSentence.getId(), textSentence); TokenSequence tokenSequence = (TokenSequence) getData(); LabelSequence labelSequence = (LabelSequence) getTarget(); for (TextToken textToken : textSentence) { String text = textToken.getText(); String label = textToken.getTag(); tokenSequence.add(new Token(text)); labelSequence.add(label); } }