@Override protected Event handleTextUnit(Event event) { ITextUnit tu = event.getTextUnit(); String r = null; if (rules != null) { r = rules.getRules(); } boolean segmentation = MimeTypeMapper.isSegmentationSupported(tu.getMimeType()); // don't trim codes for segmented formats like XLIFF as this changes the segment boundaries from the original if (!segmentation) { TextUnitUtil.simplifyCodesPostSegmentation(tu, r, params.getRemoveLeadingTrailingCodes(), params.getMergeCodes()); } return super.handleTextUnit(event); }
@Override protected ITextUnit postProcessTextUnit(ITextUnit textUnit) { TextFragment text = textUnit.getSource().getFirstContent(); text.setCodedText(text.getCodedText()); if ( codeFinder != null ) { encoderManager.updateEncoder(textUnit.getMimeType()); codeFinder.process(text); } text.setCodedText(text.getCodedText()); return textUnit; }
@Override protected ITextUnit postProcessTextUnit (ITextUnit textUnit) { // We can use getFirstPartContent() because nothing is segmented TextFragment text = textUnit.getSource().getFirstContent(); // Treat the white spaces text.setCodedText(normalizeHtmlText(text.getCodedText(), false, textUnit.preserveWhitespaces())); // Apply the in-line codes rules if needed if ( useCodeFinder ) { encoderManager.setDefaultOptions(null, encoding, lineBreak); encoderManager.updateEncoder(textUnit.getMimeType()); // IEncoder encoder = encoderManager.getEncoder(); codeFinder.process(text); // Escape inline code content //TODO: This must be put back to fix issue 431 // but the encoder is null in several test cases // List<Code> codes = text.getCodes(); // for ( Code code : codes ) { // // Escape the data of the new inline code (and only them) // if ( code.getType().equals(InlineCodeFinder.TAGTYPE) ) { // //code.setData(encoder.encode(code.getData(), EncoderContext.SKELETON)); // code.setData(encoder.encode(code.getData(), EncoderContext.INLINE)); // } // } } return textUnit; }
if (tu.getMimeType() != null) { if (tu.getMimeType().equals(MimeTypeMapper.TTX_MIME_TYPE) || tu.getMimeType().equals(MimeTypeMapper.XLIFF_MIME_TYPE)) { mergeAsSegments = true;
@Override protected ITextUnit postProcessTextUnit(ITextUnit textUnit) { TextFragment text = textUnit.getSource().getFirstContent(); String unescaped = unescape(text); text.setCodedText(unescaped); if (codeFinder != null) { encoderManager.updateEncoder(textUnit.getMimeType()); codeFinder.process(text); // Pre-emptively re-encode anything we parsed out, since it won't happen otherwise for (Code code : text.getCodes()) { code.setData(encoderManager.encode(code.getData(), EncoderContext.TEXT)); String codeDisplayText = code.getDisplayText(); if (codeDisplayText != null) { code.setDisplayText(encoderManager.encode(codeDisplayText, EncoderContext.TEXT)); } } } return textUnit; }
boolean mergeAsSegments = MimeTypeMapper.isSegmentationSupported(tuFromSkel.getMimeType()); if (tuFromSkel.getMimeType() != null && tuFromSkel.getMimeType().equals(MimeTypeMapper.XLIFF2_MIME_TYPE)) { tuFromSkel.setSource(sourceFromTarget);
if ( oriTu.getMimeType() != null ) { if ( oriTu.getMimeType().equals(MimeTypeMapper.TTX_MIME_TYPE) || oriTu.getMimeType().equals(MimeTypeMapper.XLIFF_MIME_TYPE) ) { mergeAsSegments = true;
encoderManager.updateEncoder(tu.getMimeType());