@Override public void add(CharsRef input, CharsRef output, boolean includeOrig) { // This condition follows up on the overridden analyze method. In case lenient was set to true and there was an // exception during super.analyze we return a zero-length CharsRef for that word which caused an exception. When // the synonym mappings for the words are added using the add method we skip the ones that were left empty by // analyze i.e., in the case when lenient is set we only add those combinations which are non-zero-length. The // else would happen only in the case when the input or output is empty and lenient is set, in which case we // quietly ignore it. For more details on the control-flow see SolrSynonymParser::addInternal. if (lenient == false || (input.length > 0 && output.length > 0)) { super.add(input, output, includeOrig); } }
private CharsRef parseSynonym(String line, CharsRefBuilder reuse) throws IOException { if (reuse == null) { reuse = new CharsRefBuilder(); } int start = line.indexOf('\'')+1; int end = line.lastIndexOf('\''); String text = line.substring(start, end).replace("''", "'"); return analyze(text, reuse); }
addInternal(synset, synsetSize); synsetSize = 0; synset[synsetSize] = parseSynonym(line, new CharsRefBuilder()); synsetSize++; lastSynSetID = synSetID; addInternal(synset, synsetSize); } catch (IllegalArgumentException e) { ParseException ex = new ParseException("Invalid synonym rule at line " + br.getLineNumber(), 0);
static SynonymMap.Builder getSynonymParser(Reader rulesReader, String format, boolean expand, Analyzer analyzer) throws IOException, ParseException { SynonymMap.Builder parser; if ("wordnet".equalsIgnoreCase(format)) { parser = new WordnetSynonymParser(true, expand, analyzer); ((WordnetSynonymParser) parser).parse(rulesReader); } else { parser = new SolrSynonymParser(true, expand, analyzer); ((SolrSynonymParser) parser).parse(rulesReader); } return parser; }
Analyzer analyzer = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName, Reader reader) { Tokenizer source = new FooTokenizer(reader); TokenStream filter = new FooFilter(source); //etc... WordnetSynonymParser parser = new WordnetSynonymParser(true, false, analyzer); parser.add(wordnetReader); filter = new SynonymFilter(filter, parser.build(), true); return new TokenStreamComponents(source, filter); } };
@Override public void run() { try { File synonymFile = new File(synonymFileURL.toURI()); if(synonymFile.exists() && lastModified < synonymFile.lastModified()) { Reader rulesReader = new InputStreamReader(synonymFileURL.openStream(), Charsets.UTF_8); SynonymMap.Builder parser = null; if ("wordnet".equalsIgnoreCase(format)) { parser = new WordnetSynonymParser(true, expand, analyzer); ((WordnetSynonymParser) parser).parse(rulesReader); } else { parser = new SolrSynonymParser(true, expand, analyzer); ((SolrSynonymParser) parser).parse(rulesReader); } synonymMap = parser.build(); lastModified = synonymFile.lastModified(); } } catch (Exception e) { throw new RuntimeException("could not reload synonyms file: " + e.getMessage()); } } }
private void addInternal(CharsRef synset[], int size) { if (size <= 1) { return; // nothing to do } if (expand) { for (int i = 0; i < size; i++) { for (int j = 0; j < size; j++) { add(synset[i], synset[j], false); } } } else { for (int i = 0; i < size; i++) { add(synset[i], synset[0], false); } } } }
@Override public CharsRef analyze(String text, CharsRefBuilder reuse) throws IOException { try { return super.analyze(text, reuse); } catch (IllegalArgumentException ex) { if (lenient) { logger.info("Synonym rule for [" + text + "] was ignored"); return new CharsRef(""); } else { throw ex; } } } }
parser = new WordnetSynonymParser(true, expand, analyzer); ((WordnetSynonymParser) parser).parse(rulesReader); } else { parser = new SolrSynonymParser(true, expand, analyzer);
addInternal(synset, synsetSize); synsetSize = 0; synset[synsetSize] = parseSynonym(line, synset[synsetSize]); synsetSize++; lastSynSetID = synSetID; addInternal(synset, synsetSize); } catch (IllegalArgumentException e) { ParseException ex = new ParseException("Invalid synonym rule at line " + br.getLineNumber(), 0);
@Override public void add(CharsRef input, CharsRef output, boolean includeOrig) { // This condition follows up on the overridden analyze method. In case lenient was set to true and there was an // exception during super.analyze we return a zero-length CharsRef for that word which caused an exception. When // the synonym mappings for the words are added using the add method we skip the ones that were left empty by // analyze i.e., in the case when lenient is set we only add those combinations which are non-zero-length. The // else would happen only in the case when the input or output is empty and lenient is set, in which case we // quietly ignore it. For more details on the control-flow see SolrSynonymParser::addInternal. if (lenient == false || (input.length > 0 && output.length > 0)) { super.add(input, output, includeOrig); } }
private CharsRef parseSynonym(String line, CharsRefBuilder reuse) throws IOException { if (reuse == null) { reuse = new CharsRefBuilder(); } int start = line.indexOf('\'')+1; int end = line.lastIndexOf('\''); String text = line.substring(start, end).replace("''", "'"); return analyze(text, reuse); }
parser = new WordnetSynonymParser(true, expand, analyzer); ((WordnetSynonymParser) parser).parse(rulesReader); } else { parser = new SolrSynonymParser(true, expand, analyzer);
addInternal(synset, synsetSize); synsetSize = 0; synset[synsetSize] = parseSynonym(line, new CharsRefBuilder()); synsetSize++; lastSynSetID = synSetID; addInternal(synset, synsetSize); } catch (IllegalArgumentException e) { ParseException ex = new ParseException("Invalid synonym rule at line " + br.getLineNumber(), 0);
@Override public void add(CharsRef input, CharsRef output, boolean includeOrig) { // This condition follows up on the overridden analyze method. In case lenient was set to true and there was an // exception during super.analyze we return a zero-length CharsRef for that word which caused an exception. When // the synonym mappings for the words are added using the add method we skip the ones that were left empty by // analyze i.e., in the case when lenient is set we only add those combinations which are non-zero-length. The // else would happen only in the case when the input or output is empty and lenient is set, in which case we // quietly ignore it. For more details on the control-flow see SolrSynonymParser::addInternal. if (lenient == false || (input.length > 0 && output.length > 0)) { super.add(input, output, includeOrig); } }
private CharsRef parseSynonym(String line, CharsRef reuse) throws IOException { if (reuse == null) { reuse = new CharsRef(8); } int start = line.indexOf('\'')+1; int end = line.lastIndexOf('\''); String text = line.substring(start, end).replace("''", "'"); return analyze(analyzer, text, reuse); }
private void addInternal(CharsRef synset[], int size) throws IOException { if (size <= 1) { return; // nothing to do } if (expand) { for (int i = 0; i < size; i++) { for (int j = 0; j < size; j++) { add(synset[i], synset[j], false); } } } else { for (int i = 0; i < size; i++) { add(synset[i], synset[0], false); } } } }
@Override public CharsRef analyze(String text, CharsRefBuilder reuse) throws IOException { try { return super.analyze(text, reuse); } catch (IllegalArgumentException ex) { if (lenient) { logger.info("Synonym rule for [" + text + "] was ignored"); return new CharsRef(""); } else { throw ex; } } } }
private void addInternal(CharsRef synset[], int size) { if (size <= 1) { return; // nothing to do } if (expand) { for (int i = 0; i < size; i++) { for (int j = 0; j < size; j++) { add(synset[i], synset[j], false); } } } else { for (int i = 0; i < size; i++) { add(synset[i], synset[0], false); } } } }
@Override public CharsRef analyze(String text, CharsRefBuilder reuse) throws IOException { try { return super.analyze(text, reuse); } catch (IllegalArgumentException ex) { if (lenient) { logger.info("Synonym rule for [" + text + "] was ignored"); return new CharsRef(""); } else { throw ex; } } } }