@Override public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException { super.init(source, options, env); setLossyness(options); }
@Override public IteratorOptions describeOptions() { IteratorOptions io = super.describeOptions(); io.addNamedOption(LOSSY, "if true, failed decodes are ignored. Otherwise" + " combiner will error on failed decodes (default false): <TRUE|FALSE>"); return io; }
Combiner.setColumns(setting, Collections.singletonList(new Column(MetadataSchema.ReplicationSection.COLF))); try {
@Override public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) { // TODO test Combiner newInstance; try { newInstance = this.getClass().newInstance(); } catch (Exception e) { throw new RuntimeException(e); } newInstance.setSource(getSource().deepCopy(env)); newInstance.combiners = combiners; newInstance.combineAllColumns = combineAllColumns; newInstance.isMajorCompaction = isMajorCompaction; newInstance.reduceOnFullCompactionOnly = reduceOnFullCompactionOnly; return newInstance; }
@Override public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException { // do not want to seek to the middle of a value that should be combined... Range seekRange = IteratorUtil.maximizeStartKeyTimeStamp(range); super.seek(seekRange, columnFamilies, inclusive); findTop(); if (range.getStartKey() != null) { while (hasTop() && getTopKey().equals(range.getStartKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS) && getTopKey().getTimestamp() > range.getStartKey().getTimestamp()) { // the value has a more recent time stamp, so pass it up // log.debug("skipping "+getTopKey()); next(); } while (hasTop() && range.beforeStartKey(getTopKey())) { next(); } } }
/** * Sets the topKey and topValue based on the top key of the source. If the column of the source * top key is in the set of combiners, topKey will be the top key of the source and topValue will * be the result of the reduce method. Otherwise, topKey and topValue will be unchanged. (They are * always set to null before this method is called.) */ private void findTop() throws IOException { // check if aggregation is needed if (super.hasTop()) { workKey.set(super.getTopKey()); if (combineAllColumns || combiners.contains(workKey)) { if (workKey.isDeleted()) { sawDelete(); return; } topKey = workKey; Iterator<Value> viter = new ValueIterator(getSource()); topValue = reduce(topKey, viter); while (viter.hasNext()) viter.next(); } } }
public static IteratorSetting iteratorSetting(int priority, List<IteratorSetting.Column> columns) { IteratorSetting itset = new IteratorSetting(priority, DoubleStatsCombiner.class); if (columns == null || columns.isEmpty()) Combiner.setCombineAllColumns(itset, true); else Combiner.setColumns(itset, columns); return itset; }
@Override public boolean validateOptions(Map<String,String> options) { if (!super.validateOptions(options)) return false; try { setLossyness(options); } catch (Exception e) { throw new IllegalArgumentException("bad boolean " + LOSSY + ":" + options.get(LOSSY)); } return true; }
@Override public Value multiply(Key key, Value Aval, Value Bval) { iter.reuseAndReset(Aval, Bval); return combiner.reduce(key, iter); }
/** Overridden to return false when the result of reduce is null. Marked final for safety. */ @Override public final boolean hasTop() { return super.hasTop() && super.getTopValue() != null; }
@SuppressWarnings("unchecked") @Override public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) { TypedValueCombiner<V> newInstance = (TypedValueCombiner<V>) super.deepCopy(env); newInstance.setEncoder(encoder); return newInstance; }
/** Marked final for safety. */ @Override public final Key getTopKey() { return super.getTopKey(); }
/** Marked final for safety. */ @Override public final void next() throws IOException { super.next(); }
/** Marked final for safety. */ @Override protected final SortedKeyValueIterator<Key, Value> getSource() { return super.getSource(); }
/** For use as a Combiner. Pass columns as null or empty to combine on all columns. */ public static IteratorSetting combinerSetting(int priority, List<IteratorSetting.Column> columns, ScalarOp op, ScalarType type, boolean keepZero) { IteratorSetting itset = new IteratorSetting(priority, MathTwoScalar.class); if (columns == null || columns.isEmpty()) Combiner.setCombineAllColumns(itset, true); else Combiner.setColumns(itset, columns); itset.addOptions(optionMap(op, type, null, keepZero)); // no newVisibility needed for Combiner usage return itset; }
@Override public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException { // do not want to seek to the middle of a value that should be combined... Range seekRange = IteratorUtil.maximizeStartKeyTimeStamp(range); super.seek(seekRange, columnFamilies, inclusive); findTop(); if (range.getStartKey() != null) { while (hasTop() && getTopKey().equals(range.getStartKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS) && getTopKey().getTimestamp() > range.getStartKey().getTimestamp()) { // the value has a more recent time stamp, so pass it up // log.debug("skipping "+getTopKey()); next(); } while (hasTop() && range.beforeStartKey(getTopKey())) { next(); } } }
@Override public boolean validateOptions(Map<String,String> options) { if (!super.validateOptions(options)) return false; if (options.containsKey(RADIX_OPTION) && !options.get(RADIX_OPTION).matches("\\d+")) return false; return true; }
/** * Sets the topKey and topValue based on the top key of the source. If the column of the source * top key is in the set of combiners, topKey will be the top key of the source and topValue will * be the result of the reduce method. Otherwise, topKey and topValue will be unchanged. (They are * always set to null before this method is called.) */ private void findTop() throws IOException { // check if aggregation is needed if (super.hasTop()) { workKey.set(super.getTopKey()); if (combineAllColumns || combiners.contains(workKey)) { if (workKey.isDeleted()) { sawDelete(); return; } topKey = workKey; Iterator<Value> viter = new ValueIterator(getSource()); topValue = reduce(topKey, viter); while (viter.hasNext()) viter.next(); } } }
@Override public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) { // TODO test Combiner newInstance; try { newInstance = this.getClass().newInstance(); } catch (Exception e) { throw new RuntimeException(e); } newInstance.setSource(getSource().deepCopy(env)); newInstance.combiners = combiners; newInstance.combineAllColumns = combineAllColumns; newInstance.isMajorCompaction = isMajorCompaction; newInstance.reduceOnFullCompactionOnly = reduceOnFullCompactionOnly; return newInstance; }
for (Combiner agg : aggList) { reducedValue = agg.reduce(key.getKey(), valueItr);