public TableConfiguration(ServerContext context, Table.ID tableId, NamespaceConfiguration parent) { this.context = requireNonNull(context); this.tableId = requireNonNull(tableId); this.parent = requireNonNull(parent); iteratorConfig = new EnumMap<>(IteratorScope.class); for (IteratorScope scope : IteratorScope.values()) { iteratorConfig.put(scope, new AtomicReference<>(null)); } }
/** * Generate the initial (default) properties for a table * * @param limitVersion * include a VersioningIterator at priority 20 that retains a single version of a given * K/V pair. * @return A map of Table properties */ public static Map<String,String> generateInitialTableProperties(boolean limitVersion) { TreeMap<String,String> props = new TreeMap<>(); if (limitVersion) { for (IteratorScope iterScope : IteratorScope.values()) { props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + ".vers", "20," + VersioningIterator.class.getName()); props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + ".vers.opt.maxVersions", "1"); } } props.put(Property.TABLE_CONSTRAINT_PREFIX + "1", DefaultKeySizeConstraint.class.getName()); return props; }
for (IteratorScope scope : IteratorScope.values()) { if (allScopes || cl.hasOption(scopeOpts.get(scope).getOpt())) desiredScopes.add(scope);
for (IteratorScope scope : IteratorScope.values()) { if (allScopes || cl.hasOption(scopeOpts.get(scope).getOpt())) desiredScopes.add(scope);
protected void configureGidxTable(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { // Add the UID aggregator for (IteratorScope scope : IteratorScope.values()) { String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "UIDAggregator"); setPropertyIfNecessary(tableName, stem, "19,datawave.iterators.TotalAggregatingIterator", tops, log); stem += ".opt."; setPropertyIfNecessary(tableName, stem + "*", "datawave.ingest.table.aggregator.GlobalIndexUidAggregator", tops, log); if (markingsSetupIteratorEnabled) { // we want the markings setup iterator init method to be called up front stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "MarkingsLoader"); setPropertyIfNecessary(tableName, stem, markingsSetupIteratorConfig, tops, log); } } // Set up the bloom filters for faster queries on the index portion if (enableBloomFilters) { setPropertyIfNecessary(tableName, Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), ShardIndexKeyFunctor.class.getName(), tops, log); } setPropertyIfNecessary(tableName, Property.TABLE_BLOOM_ENABLED.getKey(), Boolean.toString(enableBloomFilters), tops, log); }
protected void configureShardTable(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { // Set a text index aggregator on the "tf" (Term Frequency) column family CombinerConfiguration tfConf = new CombinerConfiguration(new Column("tf"), new IteratorSetting(10, "TF", datawave.ingest.table.aggregator.TextIndexAggregator.class.getName())); setAggregatorConfigurationIfNecessary(tableName, Collections.singletonList(tfConf), tops, log); if (markingsSetupIteratorEnabled) { for (IteratorScope scope : IteratorScope.values()) { // we want the markings setup iterator init method to be called up front String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "MarkingsLoader"); setPropertyIfNecessary(tableName, stem, markingsSetupIteratorConfig, tops, log); } } // Set the locality group for the full content column family setLocalityGroupConfigurationIfNecessary(tableName, localityGroups, tops, log); // Set up the bloom filters for faster queries on the index portion if (enableBloomFilters) { setPropertyIfNecessary(tableName, Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), SHARD_KEY_FUNCTOR_CLASS, tops, log); } setPropertyIfNecessary(tableName, Property.TABLE_BLOOM_ENABLED.getKey(), Boolean.toString(enableBloomFilters), tops, log); // Set up the table balancer for shards setPropertyIfNecessary(tableName, Property.TABLE_LOAD_BALANCER.getKey(), shardTableBalancerClass, tops, log); }
@Override public void configureTable(String table, TableOperations tops, Configuration conf) throws Exception { IteratorSetting is = tops.getIteratorSetting(table, ITER_NAME, IteratorUtil.IteratorScope.scan); String metricName = metric.toString(); if (is == null) { // create a fresh iterator Map<String,String> options = new TreeMap<>(); options.put("type", "STRING"); options.put("columns", metricName); is = new IteratorSetting(ITER_PRIORITY, ITER_NAME, SummingCombiner.class, options); tops.attachIterator(table, is); } else { // if iterator exists, piggyback on it String columns = is.getOptions().get("columns"); if (!columns.contains(metricName)) { for (IteratorUtil.IteratorScope scope : IteratorUtil.IteratorScope.values()) { String config = String.format("table.iterator.%s.%s.opt.columns", scope, ITER_NAME); tops.setProperty(table, config, columns.concat("," + metricName)); } } } }
/** * Generate the initial (default) properties for a table * * @param limitVersion * include a VersioningIterator at priority 20 that retains a single version of a given * K/V pair. * @return A map of Table properties */ public static Map<String,String> generateInitialTableProperties(boolean limitVersion) { TreeMap<String,String> props = new TreeMap<>(); if (limitVersion) { for (IteratorScope iterScope : IteratorScope.values()) { props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + ".vers", "20," + VersioningIterator.class.getName()); props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + ".vers.opt.maxVersions", "1"); } } props.put(Property.TABLE_CONSTRAINT_PREFIX.toString() + "1", DefaultKeySizeConstraint.class.getName()); return props; }
/** * Helper method for adding an option to an iterator * which OneTable and TwoTable Graphulo operations will interpret to limit the scopes an iterator applies to. * @param scopes Scopes to limit the iterator to. Only meaningful if not null, not empty, and not all. * Choices are from {@link org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope}: scan, minc, majc. * @return The same IteratorSetting as the one passed in. */ public static IteratorSetting addOnScopeOption(IteratorSetting itset, EnumSet<IteratorUtil.IteratorScope> scopes) { if (scopes == null || scopes.isEmpty() || scopes.size() == IteratorUtil.IteratorScope.values().length) return itset; String s = ""; for (IteratorUtil.IteratorScope scope : scopes) { s += scope.name() + ','; } itset.addOption(ONSCOPE_OPTION, s); return itset; }
protected void configureGridxTable(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { // Add the UID aggregator for (IteratorScope scope : IteratorScope.values()) { String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "UIDAggregator"); setPropertyIfNecessary(tableName, stem, "19,datawave.iterators.TotalAggregatingIterator", tops, log); stem += ".opt."; setPropertyIfNecessary(tableName, stem + "*", "datawave.ingest.table.aggregator.GlobalIndexUidAggregator", tops, log); if (markingsSetupIteratorEnabled) { // we want the markings setup iterator init method to be called up front stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "MarkingsLoader"); setPropertyIfNecessary(tableName, stem, markingsSetupIteratorConfig, tops, log); } } // Set up the bloom filters for faster queries on the index portion if (enableBloomFilters) { setPropertyIfNecessary(tableName, Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), ShardIndexKeyFunctor.class.getName(), tops, log); } setPropertyIfNecessary(tableName, Property.TABLE_BLOOM_ENABLED.getKey(), Boolean.toString(enableBloomFilters), tops, log); }
@Override public void configure(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { // add the SummingCombiner Iterator for each iterator scope (majc, minc, scan) for (IteratorUtil.IteratorScope scope : IteratorUtil.IteratorScope.values()) { final StringBuilder propName = new StringBuilder(String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "sum")); setPropertyIfNecessary(mTableName, propName.toString(), "19,org.apache.accumulo.core.iterators.user.SummingCombiner", tops, mLog); propName.append(".opt."); setPropertyIfNecessary(mTableName, propName + "all", "true", tops, mLog); setPropertyIfNecessary(mTableName, propName + "lossy", "FALSE", tops, mLog); setPropertyIfNecessary(mTableName, propName + "type", "STRING", tops, mLog); } // enable bloom filters if necessary. if (mEnableBloomFilters) { setPropertyIfNecessary(mTableName, Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), ShardIndexKeyFunctor.class.getName(), tops, mLog); } setPropertyIfNecessary(mTableName, Property.TABLE_BLOOM_ENABLED.getKey(), Boolean.toString(mEnableBloomFilters), tops, mLog); } }
public static Map<String,String> generateAggTableProperties(List<CombinerConfiguration> aggregators) { Map<String,String> props = new TreeMap<>(); for (IteratorScope iterScope : IteratorScope.values()) { if (!aggregators.isEmpty()) { props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + ".agg", "10," + PropogatingIterator.class.getName()); } } for (CombinerConfiguration ac : aggregators) { for (IteratorSetting.Column column : ac.getColumns()) { for (IteratorScope iterScope : IteratorScope.values()) { props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + ".agg.opt." + column.getColumnFamily(), ac.getSettings().getIteratorClass()); } } } return props; }
@Override public void configure(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { if (tableName != null) { // Add the Edge Combiner for (IteratorScope scope : IteratorScope.values()) { // Add the EdgeCombiner BELOW the Versioning iterator int combinerPriority = getVersionIteratorPriority(tops, scope) - 1; String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "EdgeCombiner"); setPropertyIfNecessary(tableName, stem, combinerPriority + ",datawave.iterators.EdgeCombiner", tops, log); setPropertyIfNecessary(tableName, stem + ".opt.all", "true", tops, log); } } }
@Override public void configure(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { for (IteratorScope scope : IteratorScope.values()) { setCombinerForLoadDateCounts(tops, scope.name()); configureToDropBadData(tops, scope.name()); } setLocalityGroupConfigurationIfNecessary(getLoadDatesTableName(conf), localityGroups, tops, log); }
@Override public void configure(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { super.configure(tops); for (IteratorUtil.IteratorScope scope : IteratorUtil.IteratorScope.values()) { configureToDropBadData(tops, scope.name()); } }
@Override public void configure(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { if (tableName != null) { for (IteratorScope scope : IteratorScope.values()) { setFrequencyCombiner(tops, scope.name()); setCombinerForCountMetadata(tops, scope.name()); setCombinerForEdgeMetadata(tops, scope.name()); } } }
public void overrideUidAggregator() throws AccumuloSecurityException, AccumuloException { for (IteratorUtil.IteratorScope scope : IteratorUtil.IteratorScope.values()) { String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "UIDAggregator"); // Override the UidAggregator with a mock aggregator to lower the UID.List MAX uid limit. connector.tableOperations().setProperty(SHARD_INDEX_TABLE_NAME, stem + ".opt.*", "datawave.query.util.InMemoryGlobalIndexUidAggregator"); connector.tableOperations().setProperty(SHARD_RINDEX_TABLE_NAME, stem + ".opt.*", "datawave.query.util.InMemoryGlobalIndexUidAggregator"); } }
private static void configureAgeOff(TableOperations tops, String tableName, int numDays) { for (IteratorUtil.IteratorScope scope : IteratorUtil.IteratorScope.values()) { String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope, AGE_OFF); try { tops.setProperty(tableName, stem + ".opt.ttl", "45"); tops.setProperty(tableName, stem, "18,datawave.iterators.filter.DateInRowAgeOffFilter"); } catch (AccumuloException | AccumuloSecurityException e) { log.error("Could not create age off configuration for " + stem, e); } } }
protected void configureDateIndexTable(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { // Add the DATE aggregator for (IteratorScope scope : IteratorScope.values()) { String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "DATEAggregator"); setPropertyIfNecessary(tableName, stem, "19,datawave.iterators.TotalAggregatingIterator", tops, log); stem += ".opt."; setPropertyIfNecessary(tableName, stem + "*", "datawave.ingest.table.aggregator.DateIndexDateAggregator", tops, log); } setPropertyIfNecessary(tableName, Property.TABLE_BLOOM_ENABLED.getKey(), Boolean.toString(false), tops, log); // Set the locality group for the full content column family setLocalityGroupConfigurationIfNecessary(tableName, localityGroups, tops, log); }
public TableConfiguration(Instance instance, String tableId, NamespaceConfiguration parent) { this.instance = instance; this.tableId = tableId; this.parent = parent; iteratorConfig = new EnumMap<>(IteratorScope.class); for (IteratorScope scope : IteratorScope.values()) { iteratorConfig.put(scope, new AtomicReference<ParsedIteratorConfig>(null)); } }