@Override public void removeSummarizers(String tableName, Predicate<SummarizerConfiguration> predicate) throws AccumuloException, TableNotFoundException, AccumuloSecurityException { Collection<SummarizerConfiguration> summarizerConfigs = SummarizerConfiguration .fromTableProperties(getProperties(tableName)); for (SummarizerConfiguration sc : summarizerConfigs) { if (predicate.test(sc)) { Set<String> ks = sc.toTableProperties().keySet(); for (String key : ks) { removeProperty(tableName, key); } } } }
public static TSummarizerConfiguration toThrift(SummarizerConfiguration sc) { return new TSummarizerConfiguration(sc.getClassName(), sc.getOptions(), sc.getPropertyId()); }
public SummarizerConfiguration build() { return new SummarizerConfiguration(className, configId, imBuilder.build()); } }
@Override public boolean shouldCompact(MajorCompactionRequest request) { Collection<SummarizerConfiguration> configuredSummarizers = SummarizerConfiguration .fromTableProperties(request.getTableProperties()); // check if delete summarizer is configured for table if (configuredSummarizers.stream().map(sc -> sc.getClassName()) .anyMatch(cn -> cn.equals(DeletesSummarizer.class.getName()))) { // This is called before gatherInformation, so need to always queue for compaction until // context // can be gathered. Also its not safe to request summary // information here as its a blocking operation. Blocking operations are not allowed in // shouldCompact. return true; } else { return super.shouldCompact(request); } }
@Override public void addSummarizers(String tableName, SummarizerConfiguration... newConfigs) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { HashSet<SummarizerConfiguration> currentConfigs = new HashSet<>( SummarizerConfiguration.fromTableProperties(getProperties(tableName))); HashSet<SummarizerConfiguration> newConfigSet = new HashSet<>(Arrays.asList(newConfigs)); newConfigSet.removeIf(currentConfigs::contains); Set<String> newIds = newConfigSet.stream().map(SummarizerConfiguration::getPropertyId) .collect(toSet()); for (SummarizerConfiguration csc : currentConfigs) { if (newIds.contains(csc.getPropertyId())) { throw new IllegalArgumentException("Summarizer property id is in use by " + csc); } } Set<Entry<String,String>> es = SummarizerConfiguration.toTableProperties(newConfigSet) .entrySet(); for (Entry<String,String> entry : es) { setProperty(tableName, entry.getKey(), entry.getValue()); } }
@Override public WriterOptions withSummarizers(SummarizerConfiguration... summarizerConf) { Objects.requireNonNull(summarizerConf); Map<String,String> tmp = SummarizerConfiguration.toTableProperties(summarizerConf); checkDisjoint(tableConfig, tmp, "summarizer"); this.summarizerProps = tmp; return this; } }
public Gatherer(ClientContext context, TSummaryRequest request, AccumuloConfiguration tableConfig, CryptoService cryptoService) { this.ctx = context; this.tableId = Table.ID.of(request.tableId); this.startRow = ByteBufferUtil.toText(request.bounds.startRow); this.endRow = ByteBufferUtil.toText(request.bounds.endRow); this.clipRange = new Range(startRow, false, endRow, true); this.summaries = request.getSummarizers().stream().map(SummarizerConfigurationUtil::fromThrift) .collect(Collectors.toSet()); this.request = request; this.cryptoService = cryptoService; this.summarizerPattern = request.getSummarizerPattern(); if (summarizerPattern != null) { Pattern pattern = Pattern.compile(summarizerPattern); // The way conf is converted to string below is documented in the API, so consider this when // making changes! summarySelector = conf -> pattern .matcher(conf.getClassName() + " " + new TreeMap<>(conf.getOptions())).matches(); if (!summaries.isEmpty()) { summarySelector = summarySelector.or(conf -> summaries.contains(conf)); } } else if (!summaries.isEmpty()) { summarySelector = conf -> summaries.contains(conf); } else { summarySelector = conf -> true; } this.factory = new SummarizerFactory(tableConfig); }
/** * Decodes table properties with the prefix {@code table.summarizer} into * {@link SummarizerConfiguration} objects. Table properties with prefixes other than * {@code table.summarizer} are ignored. */ public static Collection<SummarizerConfiguration> fromTableProperties(Map<String,String> props) { return fromTableProperties(props.entrySet()); }
public Summarizer getSummarizer(SummarizerConfiguration conf) { try { return newSummarizer(conf.getClassName()); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | IOException e) { throw new RuntimeException(e); } } }
private void init(SummarizerConfiguration conf) { maxCounters = Integer .parseInt(conf.getOptions().getOrDefault(MAX_COUNTERS_OPT, MAX_COUNTER_DEFAULT)); maxCounterKeyLen = Integer .parseInt(conf.getOptions().getOrDefault(MAX_COUNTER_LEN_OPT, MAX_CKL_DEFAULT)); ignoreDeletes = Boolean .parseBoolean(conf.getOptions().getOrDefault(INGNORE_DELETES_OPT, INGNORE_DELETES_DEFAULT)); }
private static SummarizerConfiguration readConfig(DataInputStream in) throws IOException { // read summarizer configuration String summarizerClazz = in.readUTF(); String configId = in.readUTF(); // @formatter:off org.apache.accumulo.core.client.summary.SummarizerConfiguration.Builder scb = SummarizerConfiguration.builder(summarizerClazz).setPropertyId(configId); // @formatter:on int numOpts = WritableUtils.readVInt(in); for (int i = 0; i < numOpts; i++) { String k = in.readUTF(); String v = in.readUTF(); scb.addOption(k, v); } return scb.build(); }
public static void setSummarizers(Class<?> implementingClass, Configuration conf, SummarizerConfiguration[] sumarizerConfigs) { Map<String,String> props = SummarizerConfiguration.toTableProperties(sumarizerConfigs); for (Entry<String,String> entry : props.entrySet()) { conf.set(enumToConfKey(implementingClass, Opts.ACCUMULO_PROPERTIES) + "." + entry.getKey(), entry.getValue()); } }
super.gatherInformation(request); Predicate<SummarizerConfiguration> summarizerPredicate = conf -> conf.getClassName() .equals(DeletesSummarizer.class.getName()) && conf.getOptions().isEmpty();
@Override public List<SummarizerConfiguration> listSummarizers(String tableName) throws AccumuloException, TableNotFoundException { return new ArrayList<>(SummarizerConfiguration.fromTableProperties(getProperties(tableName))); }
/** * @param summary * a summary * @param checkType * If true will try to ensure the classname from * {@link Summary#getSummarizerConfiguration()} is an instance of * {@link CountingSummarizer}. However this check can only succeed if the class is on the * classpath. For cases where the summary data needs to be used and the class is not on * the classpath, set this to false. */ public CounterSummary(Summary summary, boolean checkType) { if (checkType) { String className = summary.getSummarizerConfiguration().getClassName(); try { getClass().getClassLoader().loadClass(className).asSubclass(CountingSummarizer.class); } catch (ClassCastException e) { throw new IllegalArgumentException( className + " is not an instance of " + CountingSummarizer.class.getSimpleName(), e); } catch (ClassNotFoundException e) { throw new IllegalArgumentException( "Unable to check summary was produced by a " + CountingSummarizer.class.getSimpleName(), e); } } this.stats = summary.getStatistics(); }
private static List<SummarizerConfiguration> getSummarizerConfigsFiltered( SortedMap<String,String> sprops) { if (sprops.size() == 0) { return Collections.emptyList(); } SummarizerConfiguration.Builder builder = null; List<SummarizerConfiguration> configs = new ArrayList<>(); final int preLen = Property.TABLE_SUMMARIZER_PREFIX.getKey().length(); for (Entry<String,String> entry : sprops.entrySet()) { String k = entry.getKey().substring(preLen); String[] tokens = k.split("\\."); String id = tokens[0]; if (tokens.length == 1) { if (builder != null) { configs.add(builder.build()); } builder = SummarizerConfiguration.builder(entry.getValue()).setPropertyId(id); } else if (tokens.length == 3 || tokens[1].equals("opt")) { builder.addOption(tokens[2], entry.getValue()); } else { throw new IllegalArgumentException("Unable to parse summarizer property : " + k); } } configs.add(builder.build()); return configs; }
public void writeConfig(SummarizerConfiguration conf, DataOutputStream dos) throws IOException { // save class (and its config) used to generate summaries dos.writeUTF(conf.getClassName()); dos.writeUTF(conf.getPropertyId()); WritableUtils.writeVInt(dos, conf.getOptions().size()); for (Entry<String,String> entry : conf.getOptions().entrySet()) { dos.writeUTF(entry.getKey()); dos.writeUTF(entry.getValue()); } }
public static void setSummarizers(Class<?> implementingClass, Configuration conf, SummarizerConfiguration[] sumarizerConfigs) { Map<String,String> props = SummarizerConfiguration.toTableProperties(sumarizerConfigs); for (Entry<String,String> entry : props.entrySet()) { conf.set(enumToConfKey(implementingClass, Opts.ACCUMULO_PROPERTIES) + "." + entry.getKey(), entry.getValue()); } }
@Override public boolean shouldCompact(Entry<FileRef,DataFileValue> file, MajorCompactionRequest request) { if (!gatherCalled) { Collection<SummarizerConfiguration> configs = SummarizerConfiguration .fromTableProperties(request.getTableProperties()); return configs.size() > 0; } if (!summaryConfigured) { return false; } // Its possible the set of files could change between gather and now. So this will default to // compacting any files that are unknown. return !okFiles.contains(file.getKey()); } }
public static SummarizerConfiguration fromThrift(TSummarizerConfiguration config) { Builder builder = SummarizerConfiguration.builder(config.getClassname()); builder.setPropertyId(config.getConfigId()); builder.addOptions(config.getOptions()); return builder.build(); } }