for (IteratorScope scope : scopes) { String scopeStr = String.format("%s%s", Property.TABLE_ITERATOR_PREFIX, scope.name().toLowerCase()); String nameStr = String.format("%s.%s", scopeStr, setting.getName()); String optStr = String.format("%s.opt.", nameStr);
for (IteratorScope scope : IteratorScope.values()) { if (allScopes || cl.hasOption(scopeOpts.get(scope).getOpt())) desiredScopes.add(scope);
@Override public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException { super.init(source, options, env); scanning = IteratorScope.scan.equals(env.getIteratorScope()); if (scanning) { String auths = options.get(AUTH_OPT); if (auths != null && !auths.isEmpty()) { ve = new VisibilityEvaluator(new Authorizations(auths.getBytes(UTF_8))); visibleCache = new LRUMap(100); } } if (options.containsKey(MAX_BUFFER_SIZE_OPT)) { maxBufferSize = ConfigurationTypeHelper .getFixedMemoryAsBytes(options.get(MAX_BUFFER_SIZE_OPT)); } else { maxBufferSize = DEFAULT_MAX_BUFFER_SIZE; } parsedVisibilitiesCache = new LRUMap(100); }
String suffixSplit[] = suffix.split("\\.", 4); if (!suffixSplit[0].equals(scope.name())) { continue;
validationResult.addError("Aggregator iterator for scope " + iteratorScope.name() + " is not as expected. " + "Expected: " + requiredAggItrSetting + ", but found: " + aggItrSetting); validationResult.addError("Validator iterator for scope " + iteratorScope.name() + " is not as expected. " + "Expected: " + requiredValidatorItrSetting + ", but found: " + validatorItrSetting); validationResult.addError("The versioning iterator for scope " + iteratorScope.name() + " should not be set on the table.");
log.warn("found iterator '" + iteratorConfig.getIteratorName() + "' missing scope '" + s.name() + "', removing it and re-attaching");
for (IteratorScope scope : IteratorScope.values()) { if (allScopes || cl.hasOption(scopeOpts.get(scope).getOpt())) desiredScopes.add(scope);
for (String scope : splitD4mString(scopeStrs)) scopesToConsider.add( IteratorUtil.IteratorScope.valueOf(scope.toLowerCase())); } else scopesToConsider = EnumSet.allOf(IteratorUtil.IteratorScope.class);
for (IteratorScope scope : scopes) { String scopeStr = String.format("%s%s", Property.TABLE_ITERATOR_PREFIX, scope.name().toLowerCase()); String nameStr = String.format("%s.%s", scopeStr, setting.getName()); String optStr = String.format("%s.opt.", nameStr);
for (IteratorScope scope : scopes) { String scopeStr = String.format("%s%s", Property.TABLE_ITERATOR_PREFIX, scope.name().toLowerCase()); String nameStr = String.format("%s.%s", scopeStr, setting.getName()); String optStr = String.format("%s.opt.", nameStr);
@Override public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException { if (env != null && !IteratorScope.scan.equals(env.getIteratorScope())) { throw new IOException("This is a scan time only iterator.");
@Override public IteratorSetting getIteratorSetting(String tableName, String name, IteratorScope scope) throws AccumuloSecurityException, AccumuloException, TableNotFoundException { checkArgument(tableName != null, "tableName is null"); checkArgument(name != null, "name is null"); checkArgument(scope != null, "scope is null"); int priority = -1; String classname = null; Map<String,String> settings = new HashMap<>(); String root = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name().toLowerCase(), name); String opt = root + ".opt."; for (Entry<String,String> property : this.getProperties(tableName)) { if (property.getKey().equals(root)) { String parts[] = property.getValue().split(","); if (parts.length != 2) { throw new AccumuloException("Bad value for iterator setting: " + property.getValue()); } priority = Integer.parseInt(parts[0]); classname = parts[1]; } else if (property.getKey().startsWith(opt)) { settings.put(property.getKey().substring(opt.length()), property.getValue()); } } if (priority <= 0 || classname == null) { return null; } return new IteratorSetting(priority, name, classname, settings); }
scopeOpts.put(IteratorScope.minc, new Option(IteratorScope.minc.name(), "minor-compaction", false, "list iterator for minor compaction scope")); scopeOpts.put(IteratorScope.majc, new Option(IteratorScope.majc.name(), "major-compaction", false, "list iterator for major compaction scope")); scopeOpts.put(IteratorScope.scan, new Option(IteratorScope.scan.name(), "scan-time", false, "list iterator for scan scope"));
@Override public IteratorSetting getIteratorSetting(String namespace, String name, IteratorScope scope) throws AccumuloSecurityException, AccumuloException, NamespaceNotFoundException { if (!exists(namespace)) throw new NamespaceNotFoundException(null, namespace, null); int priority = -1; String classname = null; Map<String,String> settings = new HashMap<>(); String root = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name().toLowerCase(), name); String opt = root + ".opt."; for (Entry<String,String> property : this.getProperties(namespace)) { if (property.getKey().equals(root)) { String parts[] = property.getValue().split(","); if (parts.length != 2) { throw new AccumuloException("Bad value for iterator setting: " + property.getValue()); } priority = Integer.parseInt(parts[0]); classname = parts[1]; } else if (property.getKey().startsWith(opt)) { settings.put(property.getKey().substring(opt.length()), property.getValue()); } } if (priority <= 0 || classname == null) { return null; } return new IteratorSetting(priority, name, classname, settings); }
@Override public Options getOptions() { final Options o = new Options(); nameOpt = new Option("n", "name", true, "iterator to delete"); nameOpt.setArgName("itername"); nameOpt.setRequired(true); allScopeOpt = new Option("all", "all-scopes", false, "remove from all scopes"); mincScopeOpt = new Option(IteratorScope.minc.name(), "minor-compaction", false, "remove from minor compaction scope"); majcScopeOpt = new Option(IteratorScope.majc.name(), "major-compaction", false, "remove from major compaction scope"); scanScopeOpt = new Option(IteratorScope.scan.name(), "scan-time", false, "remove from scan scope"); OptionGroup grp = new OptionGroup(); grp.addOption(OptUtil.tableOpt("table to delete the iterator from")); grp.addOption(OptUtil.namespaceOpt("namespace to delete the iterator from")); o.addOptionGroup(grp); o.addOption(nameOpt); o.addOption(allScopeOpt); o.addOption(mincScopeOpt); o.addOption(majcScopeOpt); o.addOption(scanScopeOpt); return o; }
private Map<String, String> initializeApplicationTableProps() { Map<String, String> ntcProps = new HashMap<>(); ntcProps.put(AccumuloProps.TABLE_GROUP_PREFIX + ColumnConstants.NOTIFY_LOCALITY_GROUP_NAME, encodeColumnFamily(ColumnConstants.NOTIFY_CF)); ntcProps.put(AccumuloProps.TABLE_GROUPS_ENABLED, ColumnConstants.NOTIFY_LOCALITY_GROUP_NAME); IteratorSetting gcIter = new IteratorSetting(10, ColumnConstants.GC_CF.toString(), GarbageCollectionIterator.class); GarbageCollectionIterator.setZookeepers(gcIter, config.getAppZookeepers()); // the order relative to gc iter should not matter IteratorSetting ntfyIter = new IteratorSetting(11, ColumnConstants.NOTIFY_CF.toString(), NotificationIterator.class); for (IteratorSetting setting : new IteratorSetting[] {gcIter, ntfyIter}) { for (IteratorScope scope : EnumSet.of(IteratorUtil.IteratorScope.majc, IteratorUtil.IteratorScope.minc)) { String root = String.format("%s%s.%s", AccumuloProps.TABLE_ITERATOR_PREFIX, scope.name().toLowerCase(), setting.getName()); for (Entry<String, String> prop : setting.getOptions().entrySet()) { ntcProps.put(root + ".opt." + prop.getKey(), prop.getValue()); } ntcProps.put(root, setting.getPriority() + "," + setting.getIteratorClass()); } } return ntcProps; }
@Override public Options getOptions() { // Remove the options that specify which type of iterator this is, since // they are all scan iterators with this command. final HashSet<OptionGroup> groups = new HashSet<>(); final Options parentOptions = super.getOptions(); final Options modifiedOptions = new Options(); for (Iterator<?> it = parentOptions.getOptions().iterator(); it.hasNext();) { Option o = (Option) it.next(); if (!IteratorScope.majc.name().equals(o.getOpt()) && !IteratorScope.minc.name().equals(o.getOpt()) && !IteratorScope.scan.name().equals(o.getOpt())) { modifiedOptions.addOption(o); OptionGroup group = parentOptions.getOptionGroup(o); if (group != null) groups.add(group); } } for (OptionGroup group : groups) { modifiedOptions.addOptionGroup(group); } return modifiedOptions; }
@Override public void configureTable(String table, TableOperations tops, Configuration conf) throws Exception { IteratorSetting is = tops.getIteratorSetting(table, ITER_NAME, IteratorUtil.IteratorScope.scan); String metricName = metric.toString(); if (is == null) { // create a fresh iterator Map<String,String> options = new TreeMap<>(); options.put("type", "STRING"); options.put("columns", metricName); is = new IteratorSetting(ITER_PRIORITY, ITER_NAME, SummingCombiner.class, options); tops.attachIterator(table, is); } else { // if iterator exists, piggyback on it String columns = is.getOptions().get("columns"); if (!columns.contains(metricName)) { for (IteratorUtil.IteratorScope scope : IteratorUtil.IteratorScope.values()) { String config = String.format("table.iterator.%s.%s.opt.columns", scope, ITER_NAME); tops.setProperty(table, config, columns.concat("," + metricName)); } } } }
protected void configureShardTable(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { // Set a text index aggregator on the "tf" (Term Frequency) column family CombinerConfiguration tfConf = new CombinerConfiguration(new Column("tf"), new IteratorSetting(10, "TF", datawave.ingest.table.aggregator.TextIndexAggregator.class.getName())); setAggregatorConfigurationIfNecessary(tableName, Collections.singletonList(tfConf), tops, log); if (markingsSetupIteratorEnabled) { for (IteratorScope scope : IteratorScope.values()) { // we want the markings setup iterator init method to be called up front String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "MarkingsLoader"); setPropertyIfNecessary(tableName, stem, markingsSetupIteratorConfig, tops, log); } } // Set the locality group for the full content column family setLocalityGroupConfigurationIfNecessary(tableName, localityGroups, tops, log); // Set up the bloom filters for faster queries on the index portion if (enableBloomFilters) { setPropertyIfNecessary(tableName, Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), SHARD_KEY_FUNCTOR_CLASS, tops, log); } setPropertyIfNecessary(tableName, Property.TABLE_BLOOM_ENABLED.getKey(), Boolean.toString(enableBloomFilters), tops, log); // Set up the table balancer for shards setPropertyIfNecessary(tableName, Property.TABLE_LOAD_BALANCER.getKey(), shardTableBalancerClass, tops, log); }
@Test public void testCombinerSetOnMetadata() throws Exception { TableOperations tops = getConnector().tableOperations(); Map<String,EnumSet<IteratorScope>> iterators = tops.listIterators(MetadataTable.NAME); Assert.assertTrue(iterators.containsKey(ReplicationTableUtil.COMBINER_NAME)); EnumSet<IteratorScope> scopes = iterators.get(ReplicationTableUtil.COMBINER_NAME); Assert.assertEquals(3, scopes.size()); Assert.assertTrue(scopes.contains(IteratorScope.scan)); Assert.assertTrue(scopes.contains(IteratorScope.minc)); Assert.assertTrue(scopes.contains(IteratorScope.majc)); Iterable<Entry<String,String>> propIter = tops.getProperties(MetadataTable.NAME); HashMap<String,String> properties = new HashMap<>(); for (Entry<String,String> entry : propIter) { properties.put(entry.getKey(), entry.getValue()); } for (IteratorScope scope : scopes) { String key = Property.TABLE_ITERATOR_PREFIX.getKey() + scope.name() + "." + ReplicationTableUtil.COMBINER_NAME + ".opt.columns"; Assert.assertTrue("Properties did not contain key : " + key, properties.containsKey(key)); Assert.assertEquals(MetadataSchema.ReplicationSection.COLF.toString(), properties.get(key)); } }