@Activate protected void activate(ComponentContext ctx) throws ConfigurationException { languageConfig.setConfiguration(ctx.getProperties()); } @Deactivate
@Deactivate protected void deactivate(ComponentContext ctx){ languageConfig.setDefault(); }
/** * The {@link LanguageProcessingConfig} for the parsed language * or <code>null</code> if the language is not included in the * configuration. This will return the {@link #getDefaults()} if * the parsed language does not have a specific configuration.<p> * To obtain just language specific configuration use * {@link #getLanguageSpecificConfig(String)} * @param language the language * @return the configuration or <code>null</code> if the language is * not configured to be processed. */ public LanguageProcessingConfig getConfiguration(String language) { if(languages.isLanguage(language)){ LanguageProcessingConfig lpc = languageConfigs.get(language); return lpc == null ? defaultConfig : lpc; } else { return null; } }
@Override public void modifiedService(ServiceReference reference, Object service) { if(managerServicePid.equals(reference.getProperty(Constants.SERVICE_PID))){ return; //ignore this service! } LanguageConfiguration langConf = new LanguageConfiguration(SUPPORTED_LANUAGES, DEFAULT_LANG_CONF); try { langConf.setConfiguration(reference); ref2LangConfig.put(reference, langConf); langTokenizers.clear(); } catch (ConfigurationException e) { log.error("Unable to track ServiceReference {} becuase of invalid LanguageConfiguration(" + SUPPORTED_LANUAGES+"="+reference.getProperty(SUPPORTED_LANUAGES)+")!",e); if(ref2LangConfig.remove(reference) != null){ langTokenizers.clear(); } } }
+ "! Users need to use String[] or Collection<?> instead!"); tpc.languages.setConfiguration(configuration); Map<String,String> defaultConfig = tpc.languages.getDefaultParameters(); for(String lang : tpc.languages.getExplicitlyIncluded()){ LanguageProcessingConfig lpc = tpc.defaultConfig.clone(); applyLanguageParameter(lpc, lang, tpc.languages.getParameters(lang)); tpc.languageConfigs.put(lang, lpc);
Map<String,String> defaultParams = fstConfig.getDefaultParameters(); String fstName = defaultParams.get(IndexConfiguration.PARAM_FST); String indexField = defaultParams.get(IndexConfiguration.PARAM_FIELD); if(fstConfig.useWildcard()){ if(language != null && //successfully parsed language fstConfig.isLanguage(language) && !fstConfig.getExplicitlyIncluded().contains(language)){ for(String language : fstConfig.getExplicitlyIncluded()){ Map<String,String> config = fstConfig.getParameters(language); String langIndexField = config.get(IndexConfiguration.PARAM_FIELD); String langStoreField = config.get(IndexConfiguration.PARAM_STORE_FIELD);
@Override public String[] tokenize(String label, String language) { if(label == null){ throw new IllegalArgumentException("The parsed Label MUST NOT be NULL!"); } if(languageConfig.isLanguage(language)){ String modelName = languageConfig.getParameter(language, PARAM_MODEL); if(modelName != null){ try { TokenizerModel model = openNlp.getModel(TokenizerModel.class, modelName, null); return new TokenizerME(model).tokenize(label); } catch (Exception e) { log.warn("Unable to load configured TokenizerModel '"+modelName + "' for language '"+language + "! Fallback to default Tokenizers",e); } } //fallback to the defaults return openNlp.getTokenizer(language).tokenize(label); } else { //language not configured return null; } }
throw new IllegalArgumentException("The parsed label MUST NOT be NULL!"); if((language == null && langConf.useWildcard()) || langConf.isLanguage(language)){ if(label.isEmpty()){ return EMPTY;
private POSTagger getPOSTagger(String language) { String modelName = languageConfig.getParameter(language,MODEL_NAME_PARAM); try { POSModel model; if(modelName == null){ //use the default model = openNLP.getPartOfSpeechModel(language); } else { model = openNLP.getModel(POSModel.class, modelName, null); } if(model != null) { log.debug("POS Tagger Model {} for lanugage '{}' version: {}", new Object[]{model.getClass().getSimpleName(), model.getLanguage(), model.getVersion() != null ? model.getVersion() : "undefined"}); return new POSTaggerME(model); } } catch (Exception e) { log.warn("Unable to load POS model for language '"+language+"'!",e); } log.debug("POS tagging Model for Language '{}' not available.", language); return null; }
/** * Returns the value of the parameter for the language (if present and the * langage is active). This merges language specific parameters with * default parameters. * @param language the language * @param paramName the name of the param * @return the param or <code>null</code> if not present OR the language * is not active. */ public String getParameter(String language, String paramName) { Map<String,String> params = getParameters(language); int countrySepPos = language == null ? -1 : language.indexOf('-'); //we need to fallback to the language specific config if // * there is a country code // * no country specific params OR // * param not present in country specific config if(countrySepPos >= 2 && (params == null || !params.containsKey(paramName))) { params = getParameters(language.substring(0,countrySepPos)); } return params == null ? null : params.get(paramName); }
@Override public void modifiedService(ServiceReference reference, Object service) { if(managerServicePid.equals(reference.getProperty(Constants.SERVICE_PID))){ return; //ignore this service! } LanguageConfiguration langConf = new LanguageConfiguration(SUPPORTED_LANUAGES, DEFAULT_LANG_CONF); try { langConf.setConfiguration(reference); ref2LangConfig.put(reference, langConf); langTokenizers.clear(); } catch (ConfigurationException e) { log.error("Unable to track ServiceReference {} becuase of invalid LanguageConfiguration(" + SUPPORTED_LANUAGES+"="+reference.getProperty(SUPPORTED_LANUAGES)+")!",e); if(ref2LangConfig.remove(reference) != null){ langTokenizers.clear(); } } }
+ "! Users need to use String[] or Collection<?> instead!"); tpc.languages.setConfiguration(configuration); Map<String,String> defaultConfig = tpc.languages.getDefaultParameters(); for(String lang : tpc.languages.getExplicitlyIncluded()){ LanguageProcessingConfig lpc = tpc.defaultConfig.clone(); applyLanguageParameter(lpc, lang, tpc.languages.getParameters(lang)); tpc.languageConfigs.put(lang, lpc);
throw new IllegalArgumentException("The parsed label MUST NOT be NULL!"); if((language == null && langConf.useWildcard()) || langConf.isLanguage(language)){ if(label.isEmpty()){ return EMPTY;
private ChunkerME initChunker(String language) { isLangaugeConfigured(this,languageConfiguration,language, true); //check if the parsed language is ok String modelName = languageConfiguration.getParameter(language, MODEL_PARAM_NAME); ChunkerModel model; try { if(modelName == null){ // the default model model = openNLP.getChunkerModel(language); } else { model = openNLP.getModel(ChunkerModel.class, modelName, null); } }catch (IOException e) { log.warn("Unable to load Chunker model for language '"+language + "' (model: "+(modelName == null ? "default" : modelName)+")", e); return null; } catch (RuntimeException e){ log.warn("Error while creating ChunkerModel for language '"+language + "' (model: "+(modelName == null ? "default" : modelName)+")", e); return null; } if(model == null){ log.trace("no Chunker Model for language {}",language); return null; } else { return new ChunkerME(model); } }
/** * Returns the value of the parameter for the language (if present and the * langage is active). This merges language specific parameters with * default parameters. * @param language the language * @param paramName the name of the param * @return the param or <code>null</code> if not present OR the language * is not active. */ public String getParameter(String language, String paramName) { Map<String,String> params = getParameters(language); int countrySepPos = language == null ? -1 : language.indexOf('-'); //we need to fallback to the language specific config if // * there is a country code // * no country specific params OR // * param not present in country specific config if(countrySepPos >= 2 && (params == null || !params.containsKey(paramName))) { params = getParameters(language.substring(0,countrySepPos)); } return params == null ? null : params.get(paramName); }
/** * Activate and read the properties. Configures and initialises a POSTagger for each language configured in * CONFIG_LANGUAGES. * * @param ce the {@link org.osgi.service.component.ComponentContext} */ @Activate protected void activate(ComponentContext ce) throws ConfigurationException { log.info("activating POS tagging engine"); super.activate(ce); @SuppressWarnings("unchecked") Dictionary<String, Object> properties = ce.getProperties(); languageConfig.setConfiguration(properties); }
@Deactivate protected void deactivate(ComponentContext ctx){ resourceLoader = null; charFilterFactory = null; tokenizerFactory = null; filterFactories.clear(); langConf.setDefault(); }
@Override public Object addingService(ServiceReference reference) { if(managerServicePid.equals(reference.getProperty(Constants.SERVICE_PID))){ return null; //do not track this manager! } LanguageConfiguration langConf = new LanguageConfiguration(SUPPORTED_LANUAGES, DEFAULT_LANG_CONF); try { langConf.setConfiguration(reference); } catch (ConfigurationException e) { log.error("Unable to track ServiceReference {} becuase of invalid LanguageConfiguration(" + SUPPORTED_LANUAGES+"="+reference.getProperty(SUPPORTED_LANUAGES)+")!",e); return null; } Object service = bundleContext.getService(reference); if(service != null){ ref2LangConfig.put(reference, langConf); langTokenizers.clear(); } return service; }
/** * The {@link LanguageProcessingConfig} for the parsed language * or <code>null</code> if the language is not included in the * configuration. This will return the {@link #getDefaults()} if * the parsed language does not have a specific configuration.<p> * To obtain just language specific configuration use * {@link #getLanguageSpecificConfig(String)} * @param language the language * @return the configuration or <code>null</code> if the language is * not configured to be processed. */ public LanguageProcessingConfig getConfiguration(String language) { if(languages.isLanguage(language)){ LanguageProcessingConfig lpc = languageConfigs.get(language); return lpc == null ? defaultConfig : lpc; } else { return null; } }
String modelName = languageConfig.getParameter(language, MODEL_NAME_PARAM); if(modelName == null){ try {