public static boolean isEstimable(AggregationBuffer buffer) { if (buffer instanceof AbstractAggregationBuffer) { Class<? extends AggregationBuffer> clazz = buffer.getClass(); AggregationType annotation = AnnotationUtils.getAnnotation(clazz, AggregationType.class); return annotation != null && annotation.estimable(); } return false; }
public FunctionInfo(FunctionType functionType, String displayName, Class<? extends TableFunctionResolver> tFnCls, FunctionResource... resources) { this.functionType = functionType; this.displayName = displayName; this.tableFunctionResolver = tFnCls; PartitionTableFunctionDescription def = AnnotationUtils.getAnnotation( tableFunctionResolver, PartitionTableFunctionDescription.class); this.isInternalTableFunction = def != null && def.isInternal(); this.resources = resources; }
public static boolean isEstimable(AggregationBuffer buffer) { if (buffer instanceof AbstractAggregationBuffer) { Class<? extends AggregationBuffer> clazz = buffer.getClass(); AggregationType annotation = AnnotationUtils.getAnnotation(clazz, AggregationType.class); return annotation != null && annotation.estimable(); } return false; }
public FunctionInfo(FunctionType functionType, String displayName, Class<? extends TableFunctionResolver> tFnCls, FunctionResource... resources) { this.functionType = functionType; this.displayName = displayName; this.tableFunctionResolver = tFnCls; PartitionTableFunctionDescription def = AnnotationUtils.getAnnotation( tableFunctionResolver, PartitionTableFunctionDescription.class); this.isInternalTableFunction = def != null && def.isInternal(); this.resources = resources; }
public WindowFunctionInfo(FunctionType functionType, String functionName, GenericUDAFResolver resolver, FunctionResource[] resources) { super(functionType, functionName, resolver, resources); WindowFunctionDescription def = AnnotationUtils.getAnnotation(resolver.getClass(), WindowFunctionDescription.class); supportsWindow = def == null ? true : def.supportsWindow(); pivotResult = def == null ? false : def.pivotResult(); impliesOrder = def == null ? false : def.impliesOrder(); }
/** * Returns whether a GenericUDF is a runtime constant or not. */ public static boolean isRuntimeConstant(GenericUDF genericUDF) { UDFType genericUDFType = AnnotationUtils.getAnnotation(genericUDF.getClass(), UDFType.class); if (genericUDFType != null && genericUDFType.runtimeConstant()) { return true; } if (genericUDF instanceof GenericUDFBridge) { GenericUDFBridge bridge = (GenericUDFBridge) genericUDF; UDFType bridgeUDFType = AnnotationUtils.getAnnotation(bridge.getUdfClass(), UDFType.class); if (bridgeUDFType != null && bridgeUDFType.runtimeConstant()) { return true; } } if (genericUDF instanceof GenericUDFMacro) { GenericUDFMacro macro = (GenericUDFMacro) (genericUDF); return macro.isRuntimeConstant(); } return false; }
/** * Returns whether a GenericUDF is stateful or not. */ public static boolean isStateful(GenericUDF genericUDF) { UDFType genericUDFType = AnnotationUtils.getAnnotation(genericUDF.getClass(), UDFType.class); if (genericUDFType != null && genericUDFType.stateful()) { return true; } if (genericUDF instanceof GenericUDFBridge) { GenericUDFBridge bridge = (GenericUDFBridge) genericUDF; UDFType bridgeUDFType = AnnotationUtils.getAnnotation(bridge.getUdfClass(), UDFType.class); if (bridgeUDFType != null && bridgeUDFType.stateful()) { return true; } } if (genericUDF instanceof GenericUDFMacro) { GenericUDFMacro macro = (GenericUDFMacro) (genericUDF); return macro.isStateful(); } return false; }
public WindowFunctionInfo(FunctionType functionType, String functionName, GenericUDAFResolver resolver, FunctionResource[] resources) { super(functionType, functionName, resolver, resources); WindowFunctionDescription def = AnnotationUtils.getAnnotation(resolver.getClass(), WindowFunctionDescription.class); supportsWindow = def == null ? true : def.supportsWindow(); pivotResult = def == null ? false : def.pivotResult(); impliesOrder = def == null ? false : def.impliesOrder(); }
/** * Returns whether a GenericUDF is stateful or not. */ public static boolean isStateful(GenericUDF genericUDF) { UDFType genericUDFType = AnnotationUtils.getAnnotation(genericUDF.getClass(), UDFType.class); if (genericUDFType != null && genericUDFType.stateful()) { return true; } if (genericUDF instanceof GenericUDFBridge) { GenericUDFBridge bridge = (GenericUDFBridge) genericUDF; UDFType bridgeUDFType = AnnotationUtils.getAnnotation(bridge.getUdfClass(), UDFType.class); if (bridgeUDFType != null && bridgeUDFType.stateful()) { return true; } } if (genericUDF instanceof GenericUDFMacro) { GenericUDFMacro macro = (GenericUDFMacro) (genericUDF); return macro.isStateful(); } return false; }
/** * Use this to check if function is ranking function * * @param name * name of a function * @return true if function is a UDAF, has WindowFunctionDescription annotation and the annotations * confirms a ranking function, false otherwise * @throws SemanticException */ public static boolean isRankingFunction(String name) throws SemanticException { FunctionInfo info = getFunctionInfo(name); if (info == null) { return false; } GenericUDAFResolver res = info.getGenericUDAFResolver(); if (res == null) { return false; } WindowFunctionDescription desc = AnnotationUtils.getAnnotation(res.getClass(), WindowFunctionDescription.class); return (desc != null) && desc.rankingFunction(); }
private boolean needConversion(TableDesc tableDesc, List<PartitionDesc> partDescs) { Class<?> tableSerDe = tableDesc.getDeserializerClass(); SerDeSpec spec = AnnotationUtils.getAnnotation(tableSerDe, SerDeSpec.class); if (null == spec) { // Serde may not have this optional annotation defined in which case be conservative // and say conversion is needed. return true; } String[] schemaProps = spec.schemaProps(); Properties tableProps = tableDesc.getProperties(); for (PartitionDesc partitionDesc : partDescs) { if (!tableSerDe.getName().equals(partitionDesc.getDeserializerClassName())) { return true; } Properties partProps = partitionDesc.getProperties(); for (String schemaProp : schemaProps) { if (!org.apache.commons.lang3.StringUtils.equals( tableProps.getProperty(schemaProp), partProps.getProperty(schemaProp))) { return true; } } } return false; }
/** * Returns whether a GenericUDF is deterministic or not. */ public static boolean isDeterministic(GenericUDF genericUDF) { if (isStateful(genericUDF)) { // stateful implies non-deterministic, regardless of whatever // the deterministic annotation declares return false; } UDFType genericUDFType = AnnotationUtils.getAnnotation(genericUDF.getClass(), UDFType.class); if (genericUDFType != null && genericUDFType.deterministic() == false) { return false; } if (genericUDF instanceof GenericUDFBridge) { GenericUDFBridge bridge = (GenericUDFBridge) (genericUDF); UDFType bridgeUDFType = AnnotationUtils.getAnnotation(bridge.getUdfClass(), UDFType.class); if (bridgeUDFType != null && bridgeUDFType.deterministic() == false) { return false; } } if (genericUDF instanceof GenericUDFMacro) { GenericUDFMacro macro = (GenericUDFMacro) (genericUDF); return macro.isDeterministic(); } return true; }
/** * Checks if this grouping is like distinct, which means that all non-distinct grouping * columns behave like they were distinct - for example min and max operators. */ public boolean isDistinctLike() { ArrayList<AggregationDesc> aggregators = getAggregators(); for (AggregationDesc ad : aggregators) { if (!ad.getDistinct()) { GenericUDAFEvaluator udafEval = ad.getGenericUDAFEvaluator(); UDFType annot = AnnotationUtils.getAnnotation(udafEval.getClass(), UDFType.class); if (annot == null || !annot.distinctLike()) { return false; } } } return true; }
private boolean needConversion(TableDesc tableDesc, List<PartitionDesc> partDescs) { Class<?> tableSerDe = tableDesc.getDeserializerClass(); SerDeSpec spec = AnnotationUtils.getAnnotation(tableSerDe, SerDeSpec.class); if (null == spec) { // Serde may not have this optional annotation defined in which case be conservative // and say conversion is needed. return true; } String[] schemaProps = spec.schemaProps(); Properties tableProps = tableDesc.getProperties(); for (PartitionDesc partitionDesc : partDescs) { if (!tableSerDe.getName().equals(partitionDesc.getDeserializerClassName())) { return true; } Properties partProps = partitionDesc.getProperties(); for (String schemaProp : schemaProps) { if (!org.apache.commons.lang3.StringUtils.equals( tableProps.getProperty(schemaProp), partProps.getProperty(schemaProp))) { return true; } } } return false; }
/** * Use this to check if function is ranking function * * @param name * name of a function * @return true if function is a UDAF, has WindowFunctionDescription annotation and the annotations * confirms a ranking function, false otherwise * @throws SemanticException */ public static boolean isRankingFunction(String name) throws SemanticException { FunctionInfo info = getFunctionInfo(name); if (info == null) { return false; } GenericUDAFResolver res = info.getGenericUDAFResolver(); if (res == null) { return false; } WindowFunctionDescription desc = AnnotationUtils.getAnnotation(res.getClass(), WindowFunctionDescription.class); return (desc != null) && desc.rankingFunction(); }
/** * Returns whether a GenericUDF is deterministic or not. */ public static boolean isDeterministic(GenericUDF genericUDF) { if (isStateful(genericUDF)) { // stateful implies non-deterministic, regardless of whatever // the deterministic annotation declares return false; } UDFType genericUDFType = AnnotationUtils.getAnnotation(genericUDF.getClass(), UDFType.class); if (genericUDFType != null && genericUDFType.deterministic() == false) { return false; } if (genericUDF instanceof GenericUDFBridge) { GenericUDFBridge bridge = (GenericUDFBridge) (genericUDF); UDFType bridgeUDFType = AnnotationUtils.getAnnotation(bridge.getUdfClass(), UDFType.class); if (bridgeUDFType != null && bridgeUDFType.deterministic() == false) { return false; } } if (genericUDF instanceof GenericUDFMacro) { GenericUDFMacro macro = (GenericUDFMacro) (genericUDF); return macro.isDeterministic(); } return true; }
private void setOperatorNotSupported(Operator<? extends OperatorDesc> op) { OperatorDesc desc = op.getConf(); Annotation note = AnnotationUtils.getAnnotation(desc.getClass(), Explain.class); if (note != null) { Explain explainNote = (Explain) note; setNodeIssue(explainNote.displayName() + " (" + op.getType() + ") not supported"); } else { setNodeIssue("Operator " + op.getType() + " not supported"); } }
@Override public Void visitCall(RexCall call) { if(AnnotationUtils.getAnnotation(GenericUDFOPNotNull.class, Description.class).name().equals(call.getOperator().getName())) { if(call.getOperands().get(0) instanceof RexInputRef && !types.get(((RexInputRef)call.getOperands().get(0)).getIndex()).getType().isNullable()) { // No need to add not null filter for a constant. throw new Util.FoundOne(call); } } return super.visitCall(call); }
/** * Checks if this grouping is like distinct, which means that all non-distinct grouping * columns behave like they were distinct - for example min and max operators. */ public boolean isDistinctLike() { ArrayList<AggregationDesc> aggregators = getAggregators(); for (AggregationDesc ad : aggregators) { if (!ad.getDistinct()) { GenericUDAFEvaluator udafEval = ad.getGenericUDAFEvaluator(); UDFType annot = AnnotationUtils.getAnnotation(udafEval.getClass(), UDFType.class); if (annot == null || !annot.distinctLike()) { return false; } } } return true; }
private static long getNDVFor(ExprNodeGenericFuncDesc engfd, long numRows, Statistics parentStats) { GenericUDF udf = engfd.getGenericUDF(); if (!FunctionRegistry.isDeterministic(udf) && !FunctionRegistry.isRuntimeConstant(udf)){ return numRows; } List<Long> ndvs = Lists.newArrayList(); Class<?> udfClass = udf instanceof GenericUDFBridge ? ((GenericUDFBridge) udf).getUdfClass() : udf.getClass(); NDV ndv = AnnotationUtils.getAnnotation(udfClass, NDV.class); long udfNDV = Long.MAX_VALUE; if (ndv != null) { udfNDV = ndv.maxNdv(); } else { for (String col : engfd.getCols()) { ColStatistics stats = parentStats.getColumnStatisticsFromColName(col); if (stats != null) { ndvs.add(stats.getCountDistint()); } } } long countDistincts = ndvs.isEmpty() ? numRows : addWithExpDecay(ndvs); return Collections.min(Lists.newArrayList(countDistincts, udfNDV, numRows)); }