@Override public void process(JCas aJCas) throws AnalysisEngineProcessException { try { OutputStream outputStream = getOutputStream(aJCas, extension); /* iterate over sentences */ for (String[] line : sequenceGenerator.tokenSequences(aJCas)) { if (line.length > 0) { /* write first token */ outputStream.write(line[0].getBytes(targetEncoding)); /* write remaining tokens with token separator */ for (int i = 1; i < line.length; i++) { outputStream.write((TOKEN_SEPARATOR + line[i]).getBytes(targetEncoding)); } } outputStream.write(System.lineSeparator().getBytes(targetEncoding)); } } catch (FeaturePathException | IOException e) { throw new AnalysisEngineProcessException(e); } }
@Override public void process(JCas aJCas) throws AnalysisEngineProcessException { DocumentMetaData metadata = DocumentMetaData.get(aJCas); try { /* retrieve token sequences and convert token sequences to instances */ sequenceGenerator.tokenSequences(aJCas).stream() .map(TokenSequence::new) .map(ts -> new Instance(ts, NONE_LABEL, metadata.getDocumentId(), metadata.getDocumentUri())) .forEach(instance -> instanceList.addThruPipe(instance)); } catch (FeaturePathException e) { throw new AnalysisEngineProcessException(e); } }
List<String[]> tokenSequences = sequenceGenerator.tokenSequences(aJCas);