public ClientAggregatePlan(StatementContext context, FilterableStatement statement, TableRef table, RowProjector projector, Integer limit, Integer offset, Expression where, OrderBy orderBy, GroupBy groupBy, Expression having, QueryPlan delegate) { super(context, statement, table, projector, limit, offset, where, orderBy, delegate); this.groupBy = groupBy; this.having = having; this.clientAggregators = context.getAggregationManager().getAggregators(); // We must deserialize rather than clone based off of client aggregators because // upon deserialization we create the server-side aggregators instead of the client-side // aggregators. We use the Configuration directly here to avoid the expense of creating // another one. this.serverAggregators = ServerAggregators.deserialize(context.getScan() .getAttribute(BaseScannerRegionObserver.AGGREGATORS), context.getConnection().getQueryServices().getConfiguration(), null); // Extract hash aggregate hint, if any. HintNode hints = statement.getHint(); useHashAgg = hints != null && hints.hasHint(HintNode.Hint.HASH_AGGREGATE); }
Tuple result = useQualifierAsIndex ? new PositionBasedMultiKeyValueTuple() : new MultiKeyValueTuple(); ImmutableBytesPtr key = null; Aggregator[] rowAggregators = aggregators.getAggregators(); aggBoundary = currentKey != null && currentKey.compareTo(key) != 0; if (!aggBoundary) { aggregators.aggregate(rowAggregators, result); if (logger.isDebugEnabled()) { logger.debug(LogUtil.addCustomAnnotations( byte[] value = aggregators.toBytes(rowAggregators); Cell keyValue = PhoenixKeyValueUtil.newKeyValue(currentKey.get(), currentKey.getOffset(), aggregators.reset(rowAggregators); aggregators.aggregate(rowAggregators, result); currentKey = key; rowCount++;
@Override public Aggregator[] cache(ImmutableBytesPtr cacheKey) { ImmutableBytesPtr key = new ImmutableBytesPtr(cacheKey); Aggregator[] rowAggregators = aggregateMap.get(key); if (rowAggregators == null) { // If Aggregators not found for this distinct // value, clone our original one (we need one // per distinct value) if (logger.isDebugEnabled()) { logger.debug(LogUtil.addCustomAnnotations("Adding new aggregate bucket for row key " + Bytes.toStringBinary(key.get(), key.getOffset(), key.getLength()), customAnnotations)); } rowAggregators = aggregators.newAggregators(env.getConfiguration()); aggregateMap.put(key, rowAggregators); if (aggregateMap.size() > estDistVals) { // increase allocation estDistVals *= 1.5f; long estSize = sizeOfUnorderedGroupByMap(estDistVals, aggregators.getEstimatedByteSize()); chunk.resize(estSize); } } return rowAggregators; }
@Override public RegionScanner getScanner(final RegionScanner s) { long estSize = sizeOfUnorderedGroupByMap(aggregateMap.size(), aggregators.getEstimatedByteSize()); chunk.resize(estSize); Aggregator[] rowAggregators = entry.getValue(); byte[] value = aggregators.toBytes(rowAggregators);
@Test public void testOrderByWithNoProjection() throws SQLException { Connection conn = DriverManager.getConnection(getUrl()); try { conn.createStatement().execute("create table x (id integer primary key, A.i1 integer," + " B.i2 integer)"); Scan scan = projectQuery("select A.i1 from X group by i1 order by avg(B.i2) " + "desc"); ServerAggregators aggregators = ServerAggregators.deserialize(scan.getAttribute (BaseScannerRegionObserver.AGGREGATORS), null, null); assertEquals(2,aggregators.getAggregatorCount()); } finally { conn.close(); } }
@Test public void testCountAggregatorFirst() throws Exception { String[] queries = new String[] { "SELECT sum(2.5),organization_id FROM atable GROUP BY organization_id,entity_id", "SELECT avg(a_integer) FROM atable GROUP BY organization_id,substr(entity_id,1,3),entity_id", "SELECT count(a_string) FROM atable GROUP BY substr(organization_id,1),entity_id", "SELECT min('foo') FROM atable GROUP BY entity_id,organization_id", "SELECT min('foo'),sum(a_integer),avg(2.5),4.5,max(b_string) FROM atable GROUP BY substr(organization_id,1),entity_id", "SELECT sum(2.5) FROM atable", "SELECT avg(a_integer) FROM atable", "SELECT count(a_string) FROM atable", "SELECT min('foo') FROM atable LIMIT 5", "SELECT min('foo'),sum(a_integer),avg(2.5),4.5,max(b_string) FROM atable", }; List<Object> binds = Collections.emptyList(); String query = null; try { for (int i = 0; i < queries.length; i++) { query = queries[i]; Scan scan = compileQuery(query, binds); ServerAggregators aggregators = ServerAggregators.deserialize(scan.getAttribute(BaseScannerRegionObserver.AGGREGATORS), null, null); Aggregator aggregator = aggregators.getAggregators()[0]; assertTrue(aggregator instanceof CountAggregator); } } catch (Exception e) { throw new Exception(query, e); } }
Tuple result = new SingleKeyValueTuple(keyValue); TupleUtil.getAggregateValue(result, ptr); KeyValueSchema schema = aggregators.getValueSchema(); ValueBitSet tempValueSet = ValueBitSet.newInstance(schema); tempValueSet.clear(); SingleAggregateFunction[] funcArray = aggregators.getFunctions(); Aggregator[] sAggs = new Aggregator[funcArray.length]; Boolean hasValue;
/** * SpillManager takes care of spilling and loading tuples from spilled data structs * @param numSpillFiles * @param serverAggregators */ public SpillManager(int numSpillFiles, ServerAggregators serverAggregators, Configuration conf, SpillableGroupByCache.QueryCache cache) { try { int estValueSize = serverAggregators.getEstimatedByteSize(); spillMaps = Lists.newArrayList(); this.numSpillFiles = numSpillFiles; this.aggregators = serverAggregators; this.conf = conf; File spillFilesDir = conf.get(QueryServices.SPOOL_DIRECTORY) != null ? new File(conf.get(QueryServices.SPOOL_DIRECTORY)) : null; // Ensure that a single element fits onto a page!!! Preconditions.checkArgument(SpillFile.DEFAULT_PAGE_SIZE > estValueSize); // Create a list of spillFiles // Each Spillfile only handles up to 2GB data for (int i = 0; i < numSpillFiles; i++) { SpillFile file = SpillFile.createSpillFile(spillFilesDir); spillMaps.add(new SpillMap(file, SpillFile.DEFAULT_PAGE_SIZE, estValueSize, cache)); } } catch (IOException ioe) { throw new RuntimeException("Could not init the SpillManager"); } }
@Override public Aggregator[] newAggregators() { return newAggregators(null); }
@Override public boolean next(List<Cell> results) throws IOException { if (!cacheIter.hasNext()) { return false; } Map.Entry<ImmutableBytesWritable, Aggregator[]> ce = cacheIter.next(); ImmutableBytesWritable key = ce.getKey(); Aggregator[] aggs = ce.getValue(); byte[] value = aggregators.toBytes(aggs); if (logger.isDebugEnabled()) { logger.debug("Adding new distinct group: " + Bytes.toStringBinary(key.get(), key.getOffset(), key.getLength()) + " with aggregators " + aggs.toString() + " value = " + Bytes.toStringBinary(value)); } results.add(PhoenixKeyValueUtil.newKeyValue(key.get(), key.getOffset(), key.getLength(), SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, AGG_TIMESTAMP, value, 0, value.length)); return cacheIter.hasNext(); } };
context.getScan().setAttribute(BaseScannerRegionObserver.AGGREGATORS, ServerAggregators.serialize(aggFuncs, minNullableIndex)); ClientAggregators clientAggregators = new ClientAggregators(aggFuncs, minNullableIndex); context.getAggregationManager().setAggregators(clientAggregators);
Aggregator[] rowAggregators = groupByCache.cache(key); aggregators.aggregate(rowAggregators, result);
Tuple result = new SingleKeyValueTuple(keyValue); TupleUtil.getAggregateValue(result, ptr); KeyValueSchema schema = aggregators.getValueSchema(); ValueBitSet tempValueSet = ValueBitSet.newInstance(schema); tempValueSet.clear(); SingleAggregateFunction[] funcArray = aggregators.getFunctions(); Aggregator[] sAggs = new Aggregator[funcArray.length]; Boolean hasValue;
@Override public RegionScanner getScanner(final RegionScanner s) { long estSize = sizeOfUnorderedGroupByMap(aggregateMap.size(), aggregators.getEstimatedByteSize()); chunk.resize(estSize); Aggregator[] rowAggregators = entry.getValue(); byte[] value = aggregators.toBytes(rowAggregators);
InMemoryGroupByCache(RegionCoprocessorEnvironment env, ImmutableBytesPtr tenantId, byte[] customAnnotations, ServerAggregators aggregators, int estDistVals) { int estValueSize = aggregators.getEstimatedByteSize(); long estSize = sizeOfUnorderedGroupByMap(estDistVals, estValueSize); TenantCache tenantCache = GlobalCache.getTenantCache(env, tenantId); this.env = env; this.estDistVals = estDistVals; this.aggregators = aggregators; this.aggregateMap = Maps.newHashMapWithExpectedSize(estDistVals); this.chunk = tenantCache.getMemoryManager().allocate(estSize); this.customAnnotations = customAnnotations; }
rowAggregators = aggregators.newAggregators(env.getConfiguration()); if (logger.isDebugEnabled()) { logger.debug("Adding new aggregate bucket for row key "
private byte[] serialize(ImmutableBytesPtr key, Aggregator[] aggs, ServerAggregators serverAggs) throws IOException { DataOutputStream output = null; ByteArrayOutputStream bai = null; try { bai = new ByteArrayOutputStream(); output = new DataOutputStream(bai); // key length WritableUtils.writeVInt(output, key.getLength()); // key output.write(key.get(), key.getOffset(), key.getLength()); byte[] aggsByte = serverAggs.toBytes(aggs); // aggs length WritableUtils.writeVInt(output, aggsByte.length); // aggs output.write(aggsByte); return bai.toByteArray(); } finally { if (bai != null) { bai.close(); bai = null; } if (output != null) { output.close(); output = null; } } }
context.getScan().setAttribute(BaseScannerRegionObserver.AGGREGATORS, ServerAggregators.serialize(aggFuncs, minNullableIndex)); ClientAggregators clientAggregators = new ClientAggregators(aggFuncs, minNullableIndex); context.getAggregationManager().setAggregators(clientAggregators);
Aggregator[] rowAggregators = groupByCache.cache(key); aggregators.aggregate(rowAggregators, result);
Tuple result = useQualifierAsIndex ? new PositionBasedMultiKeyValueTuple() : new MultiKeyValueTuple(); ImmutableBytesPtr key = null; Aggregator[] rowAggregators = aggregators.getAggregators(); aggBoundary = currentKey != null && currentKey.compareTo(key) != 0; if (!aggBoundary) { aggregators.aggregate(rowAggregators, result); if (logger.isDebugEnabled()) { logger.debug(LogUtil.addCustomAnnotations( byte[] value = aggregators.toBytes(rowAggregators); Cell keyValue = PhoenixKeyValueUtil.newKeyValue(currentKey.get(), currentKey.getOffset(), aggregators.reset(rowAggregators); aggregators.aggregate(rowAggregators, result); currentKey = key; rowCount++;