private static boolean isBloomFilterAgg(AggregationDesc agg) { return "bloom_filter".equals(agg.getGenericUDAFName()); }
private boolean checkAggregator(AggregationDesc agg) throws SemanticException { if (LOG.isDebugEnabled()) { LOG.debug(String.format("Checking '%s'", agg.getExprString())); } boolean result = checkExpressions(agg.getParameters()); FunctionInfo fi = FunctionRegistry.getFunctionInfo(agg.getGenericUDAFName()); result = result && (fi != null) && fi.isNative(); if (!result) { LOG.info("Aggregator is not native: " + agg.getExprString()); } return result; }
private boolean checkAggregator(AggregationDesc agg) throws SemanticException { if (LOG.isDebugEnabled()) { LOG.debug(String.format("Checking '%s'", agg.getExprString())); } boolean result = checkExpressions(agg.getParameters()); FunctionInfo fi = FunctionRegistry.getFunctionInfo(agg.getGenericUDAFName()); result = result && (fi != null) && fi.isNative(); if (!result) { LOG.info("Aggregator is not native: " + agg.getExprString()); } return result; }
private boolean validateAggregationDesc(AggregationDesc aggDesc, GroupByDesc.Mode groupByMode, boolean hasKeys) { String udfName = aggDesc.getGenericUDAFName().toLowerCase(); if (!supportedAggregationUdfs.contains(udfName)) { setExpressionIssue("Aggregation Function", "UDF " + udfName + " not supported"); return false; } // The planner seems to pull this one out. if (aggDesc.getDistinct()) { setExpressionIssue("Aggregation Function", "DISTINCT not supported"); return false; } ArrayList<ExprNodeDesc> parameters = aggDesc.getParameters(); if (parameters != null && !validateExprNodeDesc(parameters, "Aggregation Function UDF " + udfName + " parameter")) { return false; } return true; }
GenericUDAFInfo udaf = SemanticAnalyzer.getGenericUDAFInfo(genericUDAFEvaluator, amode, aggParameters); aggregations.add(new AggregationDesc(desc.getGenericUDAFName(), udaf.genericUDAFEvaluator, udaf.convertedParameters, false, amode)); String f = SemanticAnalyzer.getColumnInternalName(aggregations.size());
List<AggregationDesc> aggrList = conf.getAggregators(); if (aggrList == null || aggrList.size() != 1 || !("count".equals(aggrList.get(0).getGenericUDAFName()))) {
private static ImmutablePair<VectorAggregationDesc,String> getVectorAggregationDesc( AggregationDesc aggrDesc, VectorizationContext vContext) throws HiveException { String aggregateName = aggrDesc.getGenericUDAFName(); List<ExprNodeDesc> parameterList = aggrDesc.getParameters(); final int parameterCount = parameterList.size(); final GenericUDAFEvaluator.Mode udafEvaluatorMode = aggrDesc.getMode(); /* * Look at evaluator to get output type info. */ GenericUDAFEvaluator evaluator = aggrDesc.getGenericUDAFEvaluator(); ObjectInspector[] parameterObjectInspectors = new ObjectInspector[parameterCount]; for (int i = 0; i < parameterCount; i++) { TypeInfo typeInfo = parameterList.get(i).getTypeInfo(); parameterObjectInspectors[i] = TypeInfoUtils .getStandardWritableObjectInspectorFromTypeInfo(typeInfo); } // The only way to get the return object inspector (and its return type) is to // initialize it... ObjectInspector returnOI = evaluator.init( aggrDesc.getMode(), parameterObjectInspectors); final TypeInfo outputTypeInfo = TypeInfoUtils.getTypeInfoFromTypeString(returnOI.getTypeName()); return getVectorAggregationDesc( aggregateName, parameterList, evaluator, outputTypeInfo, udafEvaluatorMode, vContext); }
aggParameters); String f = SemanticAnalyzer.getColumnInternalName(aggregations.size()); aggregations.add(new AggregationDesc(desc.getGenericUDAFName(), udaf.genericUDAFEvaluator, udaf.convertedParameters, false, amode)); outputColumnNames.add(f);
String aggregateName = desc.getGenericUDAFName(); VectorExpressionDescriptor.ArgumentType inputType = VectorExpressionDescriptor.ArgumentType.NONE;
private Pair<Boolean,Boolean> validateAggregationDesc(AggregationDesc aggDesc, ProcessingMode processingMode, boolean hasKeys) { String udfName = aggDesc.getGenericUDAFName().toLowerCase(); if (!supportedAggregationUdfs.contains(udfName)) { setExpressionIssue("Aggregation Function", "UDF " + udfName + " not supported");
AggregationDesc aggr = mGby.getConf().getAggregators().get(pos); if (aggr.getDistinct()) { if (indexOfDist != -1 || !aggr.getGenericUDAFName().equalsIgnoreCase("count")) {
String udafName = agg.getGenericUDAFName(); if (!(reduceSideGop && expr.startsWith(udafName))) { sb.setLength(0); // reset the buffer
String udafName = agg.getGenericUDAFName(); if (!(reduceSideGop && expr.startsWith(udafName))) { sb.setLength(0); // reset the buffer
boolean removeSemiJoin = false; for (AggregationDesc agg : aggregationDescs) { if (agg.getGenericUDAFName() != "bloom_filter") { continue;
private static Pair<GroupByDesc,VectorGroupByDesc> buildGroupByDescCountStar( VectorizationContext ctx) { AggregationDesc agg = buildAggregationDescCountStar(ctx); ArrayList<AggregationDesc> aggs = new ArrayList<AggregationDesc>(); aggs.add(agg); ArrayList<String> outputColumnNames = new ArrayList<String>(); outputColumnNames.add("_col0"); GroupByDesc desc = new GroupByDesc(); VectorGroupByDesc vectorDesc = new VectorGroupByDesc(); vectorDesc.setVecAggrDescs( new VectorAggregationDesc[] { new VectorAggregationDesc( agg.getGenericUDAFName(), new GenericUDAFCount.GenericUDAFCountEvaluator(), agg.getMode(), null, ColumnVector.Type.NONE, null, TypeInfoFactory.longTypeInfo, ColumnVector.Type.LONG, VectorUDAFCountStar.class)}); vectorDesc.setProcessingMode(VectorGroupByDesc.ProcessingMode.HASH); desc.setOutputColumnNames(outputColumnNames); desc.setAggregators(aggs); return new Pair<GroupByDesc,VectorGroupByDesc>(desc, vectorDesc); }
FunctionRegistry.getGenericUDAFResolver(aggr.getGenericUDAFName()); if (udaf instanceof GenericUDAFSum) { + aggr.getGenericUDAFName()); return null;
FunctionRegistry.getGenericUDAFResolver(aggr.getGenericUDAFName()); if (udaf instanceof GenericUDAFSum) { + aggr.getGenericUDAFName()); return null;
new VectorAggregationDesc[] { new VectorAggregationDesc( aggDesc.getGenericUDAFName(), new GenericUDAFSum.GenericUDAFSumLong(), aggDesc.getMode(),
private boolean validateAggregationDesc(AggregationDesc aggDesc, boolean isReduce) { String udfName = aggDesc.getGenericUDAFName().toLowerCase(); if (!supportedAggregationUdfs.contains(udfName)) { LOG.info("Cannot vectorize groupby aggregate expression: UDF " + udfName + " not supported"); return false; } if (aggDesc.getParameters() != null && !validateExprNodeDesc(aggDesc.getParameters())) { LOG.info("Cannot vectorize groupby aggregate expression: UDF parameters not supported"); return false; } // See if we can vectorize the aggregation. try { VectorizationContext vc = new ValidatorVectorizationContext(); if (vc.getAggregatorExpression(aggDesc, isReduce) == null) { // TODO: this cannot happen - VectorizationContext throws in such cases. LOG.info("getAggregatorExpression returned null"); return false; } } catch (Exception e) { LOG.info("Failed to vectorize", e); return false; } return true; }
String aggregateName = desc.getGenericUDAFName(); VectorExpressionDescriptor.ArgumentType inputType = VectorExpressionDescriptor.ArgumentType.NONE;