@Override public DictionaryConnection openConnection(final DataCleanerConfiguration configuration) { final ResourceConverter rc = new ResourceConverter(configuration); final Resource resource = rc.fromString(Resource.class, _filename); final Set<String> values = resource.read(in -> { final Set<String> values1 = new HashSet<>(); final BufferedReader reader = FileHelper.getBufferedReader(in, getEncoding()); try { String line = reader.readLine(); while (line != null) { if (!_caseSensitive) { line = line.toLowerCase(); } values1.add(line); line = reader.readLine(); } } catch (final IOException e) { logger.error("Failed to read line from resource: {}", resource, e); } finally { FileHelper.safeClose(reader); } return values1; }); final SimpleDictionary simpleDictionary = new SimpleDictionary(getName(), values, _caseSensitive); return simpleDictionary.openConnection(configuration); }
public SimpleDictionary loadIntoMemory(final DatastoreConnection datastoreConnection) { final DataContext dataContext = datastoreConnection.getDataContext(); final Column column = getColumn(datastoreConnection); final Query query = dataContext.query().from(column.getTable()).select(column).toQuery(); if (datastoreConnection.getDatastore().getPerformanceCharacteristics().isQueryOptimizationPreferred()) { query.getSelectClause().setDistinct(true); } final Set<String> values = new HashSet<>(); try (DataSet dataSet = dataContext.executeQuery(query)) { while (dataSet.next()) { final Object value = dataSet.getRow().getValue(0); if (value != null) { values.add(value.toString()); } } } return new SimpleDictionary(getName(), values); }
final SimpleDictionary dict = new SimpleDictionary(name1, caseSensitive, values.split("\n"));
final SimpleDictionary dict = new SimpleDictionary(name, values, caseSensitive); dict.setDescription(vldt.getDescription()); dictionaryList.add(dict);