@Override public Aggregator newServerAggregator(Configuration conf) { return new DistinctValueWithCountServerAggregator(conf); }
@Override public int getSize() { return super.getSize() + SizedUtil.ARRAY_SIZE + countMapHeapSize(); }
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { // This serializes the Map. The format is as follows // Map size(VInt ie. 1 to 5 bytes) + // ( key length [VInt ie. 1 to 5 bytes] + key bytes + value [VInt ie. 1 to 5 bytes] )* int serializationSize = countMapSerializationSize(); buffer = new byte[serializationSize]; int offset = 1; offset += ByteUtil.vintToBytes(buffer, offset, this.valueVsCount.size()); for (Entry<ImmutableBytesPtr, Integer> entry : this.valueVsCount.entrySet()) { ImmutableBytesPtr key = entry.getKey(); offset += ByteUtil.vintToBytes(buffer, offset, key.getLength()); System.arraycopy(key.get(), key.getOffset(), buffer, offset, key.getLength()); offset += key.getLength(); offset += ByteUtil.vintToBytes(buffer, offset, entry.getValue().intValue()); } if (serializationSize > compressThreshold) { // The size for the map serialization is above the threshold. We will do the Snappy compression here. byte[] compressed = new byte[COMPRESS_MARKER.length + Snappy.maxCompressedLength(buffer.length)]; System.arraycopy(COMPRESS_MARKER, 0, compressed, 0, COMPRESS_MARKER.length); int compressedLen = Snappy.compress(buffer, 1, buffer.length - 1, compressed, COMPRESS_MARKER.length); ptr.set(compressed, 0, compressedLen + 1); return true; } ptr.set(buffer, 0, offset); return true; }
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { // This serializes the Map. The format is as follows // Map size(VInt ie. 1 to 5 bytes) + // ( key length [VInt ie. 1 to 5 bytes] + key bytes + value [VInt ie. 1 to 5 bytes] )* int serializationSize = countMapSerializationSize(); buffer = new byte[serializationSize]; int offset = 1; offset += ByteUtil.vintToBytes(buffer, offset, this.valueVsCount.size()); for (Entry<ImmutableBytesPtr, Integer> entry : this.valueVsCount.entrySet()) { ImmutableBytesPtr key = entry.getKey(); offset += ByteUtil.vintToBytes(buffer, offset, key.getLength()); System.arraycopy(key.get(), key.getOffset(), buffer, offset, key.getLength()); offset += key.getLength(); offset += ByteUtil.vintToBytes(buffer, offset, entry.getValue().intValue()); } if (serializationSize > compressThreshold) { // The size for the map serialization is above the threshold. We will do the Snappy compression here. byte[] compressed = new byte[COMPRESS_MARKER.length + Snappy.maxCompressedLength(buffer.length)]; System.arraycopy(COMPRESS_MARKER, 0, compressed, 0, COMPRESS_MARKER.length); int compressedLen = Snappy.compress(buffer, 1, buffer.length - 1, compressed, COMPRESS_MARKER.length); ptr.set(compressed, 0, compressedLen + 1); return true; } ptr.set(buffer, 0, offset); return true; }
@Override public Aggregator newServerAggregator(Configuration conf) { return new DistinctValueWithCountServerAggregator(conf); }
@Override public int getSize() { return super.getSize() + SizedUtil.ARRAY_SIZE + countMapHeapSize(); }
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { // This serializes the Map. The format is as follows // Map size(VInt ie. 1 to 5 bytes) + // ( key length [VInt ie. 1 to 5 bytes] + key bytes + value [VInt ie. 1 to 5 bytes] )* int serializationSize = countMapSerializationSize(); buffer = new byte[serializationSize]; int offset = 1; offset += ByteUtil.vintToBytes(buffer, offset, this.valueVsCount.size()); for (Entry<ImmutableBytesPtr, Integer> entry : this.valueVsCount.entrySet()) { ImmutableBytesPtr key = entry.getKey(); offset += ByteUtil.vintToBytes(buffer, offset, key.getLength()); System.arraycopy(key.get(), key.getOffset(), buffer, offset, key.getLength()); offset += key.getLength(); offset += ByteUtil.vintToBytes(buffer, offset, entry.getValue().intValue()); } if (serializationSize > compressThreshold) { // The size for the map serialization is above the threshold. We will do the Snappy compression here. byte[] compressed = new byte[COMPRESS_MARKER.length + Snappy.maxCompressedLength(buffer.length)]; System.arraycopy(COMPRESS_MARKER, 0, compressed, 0, COMPRESS_MARKER.length); int compressedLen = Snappy.compress(buffer, 1, buffer.length - 1, compressed, COMPRESS_MARKER.length); ptr.set(compressed, 0, compressedLen + 1); return true; } ptr.set(buffer, 0, offset); return true; }
@Override public Aggregator newServerAggregator(Configuration conf) { return new DistinctValueWithCountServerAggregator(conf); }
@Override public int getSize() { return super.getSize() + SizedUtil.ARRAY_SIZE + countMapHeapSize(); }
@Override public Aggregator newServerAggregator(Configuration conf) { return new DistinctValueWithCountServerAggregator(conf); }
@Override public Aggregator newServerAggregator(Configuration conf) { return new DistinctValueWithCountServerAggregator(conf); }
@Override public Aggregator newServerAggregator(Configuration conf) { return new DistinctValueWithCountServerAggregator(conf); }
@Override public Aggregator newServerAggregator(Configuration conf) { return new DistinctValueWithCountServerAggregator(conf); }
@Override public Aggregator newServerAggregator(Configuration config, ImmutableBytesWritable ptr) { DistinctValueWithCountClientAggregator clientAgg = newClientAggregator(); clientAgg.aggregate(null, ptr); return new DistinctValueWithCountServerAggregator(config, clientAgg); } }
@Override public Aggregator newServerAggregator(Configuration config, ImmutableBytesWritable ptr) { DistinctCountClientAggregator clientAgg = newClientAggregator(); clientAgg.aggregate(null, ptr); return new DistinctValueWithCountServerAggregator(config, clientAgg); } }
@Override public Aggregator newServerAggregator(Configuration conf) { return new DistinctValueWithCountServerAggregator(conf); }
@Override public Aggregator newServerAggregator(Configuration conf) { return new DistinctValueWithCountServerAggregator(conf); }
@Override public Aggregator newServerAggregator(Configuration conf) { return new DistinctValueWithCountServerAggregator(conf); }
@Override public Aggregator newServerAggregator(Configuration conf) { return new DistinctValueWithCountServerAggregator(conf); }
@Override public Aggregator newServerAggregator(Configuration conf) { return new DistinctValueWithCountServerAggregator(conf); }