public SentenceSample read() throws IOException { StringBuilder sentencesString = new StringBuilder(); List<Span> sentenceSpans = new LinkedList<>(); String sentence; while ((sentence = samples.read()) != null && !sentence.equals("")) { int begin = sentencesString.length(); sentence = sentence.trim(); sentence = replaceNewLineEscapeTags(sentence); sentencesString.append(sentence); int end = sentencesString.length(); sentenceSpans.add(new Span(begin, end)); sentencesString.append(' '); } if (sentenceSpans.size() > 0) { return new SentenceSample(sentencesString.toString(), sentenceSpans.toArray(new Span[sentenceSpans.size()])); } return null; } }
String eosString = SentenceSampleStream.replaceNewLineEscapeTags(params.getEosChars()); eos = eosString.toCharArray();
String eosString = SentenceSampleStream.replaceNewLineEscapeTags( params.getEosChars()); eos = eosString.toCharArray();
public SentenceSample read() throws IOException { StringBuilder sentencesString = new StringBuilder(); List<Span> sentenceSpans = new LinkedList<>(); String sentence; while ((sentence = samples.read()) != null && !sentence.equals("")) { int begin = sentencesString.length(); sentence = sentence.trim(); sentence = replaceNewLineEscapeTags(sentence); sentencesString.append(sentence); int end = sentencesString.length(); sentenceSpans.add(new Span(begin, end)); sentencesString.append(' '); } if (sentenceSpans.size() > 0) { return new SentenceSample(sentencesString.toString(), sentenceSpans.toArray(new Span[sentenceSpans.size()])); } return null; } }
public SentenceSample read() throws IOException { StringBuilder sentencesString = new StringBuilder(); List<Span> sentenceSpans = new LinkedList<>(); String sentence; while ((sentence = samples.read()) != null && !sentence.equals("")) { int begin = sentencesString.length(); sentence = sentence.trim(); sentence = replaceNewLineEscapeTags(sentence); sentencesString.append(sentence); int end = sentencesString.length(); sentenceSpans.add(new Span(begin, end)); sentencesString.append(' '); } if (sentenceSpans.size() > 0) { return new SentenceSample(sentencesString.toString(), sentenceSpans.toArray(new Span[sentenceSpans.size()])); } return null; } }
String eosString = SentenceSampleStream.replaceNewLineEscapeTags(params.getEosChars()); eos = eosString.toCharArray();
String eosString = SentenceSampleStream.replaceNewLineEscapeTags(params.getEosChars()); eos = eosString.toCharArray();
String eosString = SentenceSampleStream.replaceNewLineEscapeTags( params.getEosChars()); eos = eosString.toCharArray();
String eosString = SentenceSampleStream.replaceNewLineEscapeTags( params.getEosChars()); eos = eosString.toCharArray();