private void setNormalizedForm(BaseToken tokenAnnotation, String token) throws AnalysisEngineProcessException { // apply LVG processing to get canonical form String normalizedForm = null; if (useCmdCache) { normalizedForm = (String) normCacheMap.get(token); if (normalizedForm == null) { // logger.info("["+ word+ "] was not found in LVG norm cache."); } } // only apply LVG processing if not found in cache first if (normalizedForm == null) { try { String out = lvgCmd.MutateToString(token); String[] output = out.split("\\|"); if ((output != null) && (output.length >= 2) && (!output[1].matches("No Output"))) { normalizedForm = output[1]; } } catch (Exception e) { throw new AnalysisEngineProcessException(e); } } if (normalizedForm != null) { tokenAnnotation.setNormalizedForm(normalizedForm); } }
private void setNormalizedForm(BaseToken tokenAnnotation, String token) throws AnalysisEngineProcessException { // apply LVG processing to get canonical form String normalizedForm = null; if (useCmdCache) { normalizedForm = (String) normCacheMap.get(token); if (normalizedForm == null) { // logger.info("["+ word+ "] was not found in LVG norm cache."); } } // only apply LVG processing if not found in cache first if (normalizedForm == null) { try { String out = lvgCmd.MutateToString(token); String[] output = out.split("\\|"); if ((output != null) && (output.length >= 2) && (!output[1].matches("No Output"))) { normalizedForm = output[1]; } } catch (Exception e) { throw new AnalysisEngineProcessException(e); } } if (normalizedForm != null) { tokenAnnotation.setNormalizedForm(normalizedForm); } }
BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.setNormalizedForm(tokens[2]); btoken.addToIndexes(); documentText.append(tokens[1] + " "); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.setNormalizedForm(tokens[2]); btoken.setPartOfSpeech(tokens[3]); btoken.addToIndexes(); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.setNormalizedForm(tokens[2]); btoken.setPartOfSpeech(tokens[4]); btoken.addToIndexes();
BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.setNormalizedForm(tokens[2]); btoken.addToIndexes(); documentText.append(tokens[1] + " "); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.setNormalizedForm(tokens[2]); btoken.setPartOfSpeech(tokens[3]); btoken.addToIndexes(); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.setNormalizedForm(tokens[2]); btoken.setPartOfSpeech(tokens[4]); btoken.addToIndexes();