/** * @return The total number of files that had some kind of issue which would cause summary * statistics to be inaccurate. This is the sum of {@link #getMissing()}, * {@link #getExtra()}, {@link #getLarge()}, and {@link #getDeleted()}. */ public long getInaccurate() { return getMissing() + getExtra() + getLarge() + getDeleted(); }
/** * @return the sum of {@link #getTooLong()} and {@link #getTooLong()} */ public long getIgnored() { return getTooLong() + getTooMany(); }
/** * Call this method to initiate a chain of fluent method calls to a create an immutable * {@link SummarizerConfiguration} * * @param className * The fully qualified name of a class that implements {@link Summarizer}. */ public static Builder builder(String className) { return new Builder(className); }
public static TSummarizerConfiguration toThrift(SummarizerConfiguration sc) { return new TSummarizerConfiguration(sc.getClassName(), sc.getOptions(), sc.getPropertyId()); }
public static SummarizerConfiguration fromThrift(TSummarizerConfiguration config) { Builder builder = SummarizerConfiguration.builder(config.getClassname()); builder.setPropertyId(config.getConfigId()); builder.addOptions(config.getOptions()); return builder.build(); } }
private static SummarizerConfiguration readConfig(DataInputStream in) throws IOException { // read summarizer configuration String summarizerClazz = in.readUTF(); String configId = in.readUTF(); // @formatter:off org.apache.accumulo.core.client.summary.SummarizerConfiguration.Builder scb = SummarizerConfiguration.builder(summarizerClazz).setPropertyId(configId); // @formatter:on int numOpts = WritableUtils.readVInt(in); for (int i = 0; i < numOpts; i++) { String k = in.readUTF(); String v = in.readUTF(); scb.addOption(k, v); } return scb.build(); }
@Override public WriterOptions withSummarizers(SummarizerConfiguration... summarizerConf) { Objects.requireNonNull(summarizerConf); Map<String,String> tmp = SummarizerConfiguration.toTableProperties(summarizerConf); checkDisjoint(tableConfig, tmp, "summarizer"); this.summarizerProps = tmp; return this; } }
public void merge(MergedSummary other, SummarizerConfiguration config, SummarizerFactory factory) { if (summary == null && other.summary != null) { summary = new HashMap<>(other.summary); } else if (summary != null && other.summary != null) { Summarizer summarizer = factory.getSummarizer(config); summarizer.combiner(config).merge(summary, other.summary); } filesContaining += other.filesContaining; filesExceedingBoundry += other.filesExceedingBoundry; filesLarge += other.filesLarge; }
@Override public void summarize(StatisticConsumer sc) { keyStats.summarize("key", sc); rowStats.summarize("row", sc); familyStats.summarize("family", sc); qualifierStats.summarize("qualifier", sc); visibilityStats.summarize("visibility", sc); valueStats.summarize("value", sc); sc.accept("total", total); } };
/** * Decodes table properties with the prefix {@code table.summarizer} into * {@link SummarizerConfiguration} objects. Table properties with prefixes other than * {@code table.summarizer} are ignored. */ public static Collection<SummarizerConfiguration> fromTableProperties(Map<String,String> props) { return fromTableProperties(props.entrySet()); }
public LgBuilder(SummarizerConfiguration conf, Summarizer kvs, String name) { this.conf = conf; this.summarizer = kvs; this.name = name; this.collector = kvs.collector(conf); }
public SummarizerConfiguration build() { return new SummarizerConfiguration(className, configId, imBuilder.build()); } }
/** * Adds an option that Summarizers can use when constructing Collectors and Combiners. * * @return this * * @see SummarizerConfiguration#getOptions() */ public Builder addOption(String key, long value) { return addOption(key, Long.toString(value)); }
public Summary(Map<String,Long> summary, SummarizerConfiguration config, long totalFiles, long filesMissingSummary, long filesWithExtra, long filesWithLarge, long deletedFiles) { this.statistics = ImmutableMap.copyOf(summary); this.config = config; this.fileStats = new FileStatistics(totalFiles, filesMissingSummary, filesWithExtra, filesWithLarge, deletedFiles); }
@Override public void accept(Key k, Value v) { seen++; if (ignoreDeletes && k.isDeleted()) { deleted++; } else { converter.convert(k, v, this::incrementCounter); } }
private List<SummaryInfo> merge(int end) { List<SummaryInfo> mergedSummaries = new ArrayList<>(); for (int i = 0; i < end; i += 2) { int mergedCount = summaries.get(i).count + summaries.get(i + 1).count; summarizer.combiner(conf).merge(summaries.get(i).summary, summaries.get(i + 1).summary); mergedSummaries.add(new SummaryInfo(summaries.get(i + 1).getLastRow(), summaries.get(i).summary, mergedCount)); } return mergedSummaries; }
/** * @see #builder(String) */ public static Builder builder(Class<? extends Summarizer> clazz) { return new Builder(clazz.getName()); } }
public LgBuilder(SummarizerConfiguration conf, Summarizer kvs) { this.conf = conf; this.summarizer = kvs; this.name = "<DEFAULT>"; this.collector = kvs.collector(conf); }
void summarize(String prefix, StatisticConsumer sc) { sc.accept(prefix + ".min", (min != Long.MAX_VALUE ? min : 0)); sc.accept(prefix + ".max", (max != Long.MIN_VALUE ? max : 0)); sc.accept(prefix + ".sum", sum); for (int i = 0; i < counts.length; i++) { if (counts[i] > 0) { sc.accept(prefix + ".logHist." + i, counts[i]); } } }