How to use

StanfordTokenizer

in

org.jdmp.stanfordpos

public static StanfordTokenizer getInstance() {
  if (instance == null) {
    try {
      instance = new StanfordTokenizer();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
  return instance;
}

String text = "";
ListMatrix<ListMatrix<String>> originalText = StanfordTokenizer.getInstance().tokenize(text);
ListMatrix<ListMatrix<MapMatrix<String, String>>> trainingText = new DefaultListMatrix<ListMatrix<MapMatrix<String, String>>>();

@BeforeClass
public static void setUp() {
  try {
    tokenizer = new StanfordTokenizer();
    tagger = new StanfordTagger();
  } catch (Exception e) {
    e.printStackTrace();
  }
}

  @Test
  public void testTokenizer() throws Exception {
    ListDataSet ds = new DefaultListDataSet();
    Sample sa1 = new DefaultSample();
    sa1.put(Sample.INPUT, s1);
    sa1.setId("sample1");
    Sample sa2 = new DefaultSample();
    sa2.put(Sample.INPUT, s2);
    sa2.setId("sample2");
    ds.add(sa1);
    ds.add(sa2);

    Tokenizer t = new StanfordTokenizer();
    t.tokenize(Sample.INPUT, ds);

    sa1 = ds.get(0);
    sa2 = ds.get(1);

    Matrix m1 = sa1.getAsMatrix(Tokenizer.TOKENIZED);
    Matrix m2 = sa2.getAsMatrix(Tokenizer.TOKENIZED);

    assertEquals(1, m1.getColumnCount());
    assertEquals(11, m1.getRowCount());
    assertEquals(1, m2.getColumnCount());
    assertEquals(5, m2.getRowCount());
  }
}