Refine search
/** * UDFRowSequence. */ @Description(name = "row_sequence", value = "_FUNC_() - Returns a generated row sequence number starting from 1") @UDFType(deterministic = false, stateful = true) public class UDFRowSequence extends UDF { private LongWritable result = new LongWritable(); public UDFRowSequence() { result.set(0); } public LongWritable evaluate() { result.set(result.get() + 1); return result; } }
private <I> void register(Class<? extends I> clazz, ArrayListMultimap<String, Class<? extends I>> methods) { Description desc = clazz.getAnnotation(Description.class); Stream<String> namesStream; if (desc != null) { namesStream = Stream.of(desc.name().split(",")) .map(String::trim); } else { namesStream = Stream.of(clazz) .map(Class::getName) .map(name -> name.replace('.', '_')); } // Checks specified array of function names whether they should be replaced // using FUNCTION_REPLACE_MAP map. namesStream.map(String::toLowerCase) .map(functionName -> FUNCTION_REPLACE_MAP.getOrDefault(functionName, functionName)) .forEach(name -> methods.put(name, clazz)); UDFType type = clazz.getAnnotation(UDFType.class); if (type != null && !type.deterministic()) { nonDeterministicUDFs.add(clazz); } }
/** * Returns whether a GenericUDF is stateful or not. */ public static boolean isStateful(GenericUDF genericUDF) { UDFType genericUDFType = AnnotationUtils.getAnnotation(genericUDF.getClass(), UDFType.class); if (genericUDFType != null && genericUDFType.stateful()) { return true; } if (genericUDF instanceof GenericUDFBridge) { GenericUDFBridge bridge = (GenericUDFBridge) genericUDF; UDFType bridgeUDFType = AnnotationUtils.getAnnotation(bridge.getUdfClass(), UDFType.class); if (bridgeUDFType != null && bridgeUDFType.stateful()) { return true; } } if (genericUDF instanceof GenericUDFMacro) { GenericUDFMacro macro = (GenericUDFMacro) (genericUDF); return macro.isStateful(); } return false; }
@UDFType(deterministic = false) public class GenericUDFBucketNumber extends GenericUDF{ @Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { return PrimitiveObjectInspectorFactory.writableStringObjectInspector; } @Override public String getDisplayString(String[] children) { return "_bucket_number"; } @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { return null; } }
names = desc.name().split(","); for (int i=0; i<names.length; i++) { names[i] = names[i].trim(); value = desc.value(); extended = desc.extended(); builder.setDeterministic(type.deterministic());
/** * Returns whether a GenericUDF is deterministic or not. */ public static boolean isDeterministic(GenericUDF genericUDF) { if (isStateful(genericUDF)) { // stateful implies non-deterministic, regardless of whatever // the deterministic annotation declares return false; } UDFType genericUDFType = AnnotationUtils.getAnnotation(genericUDF.getClass(), UDFType.class); if (genericUDFType != null && genericUDFType.deterministic() == false) { return false; } if (genericUDF instanceof GenericUDFBridge) { GenericUDFBridge bridge = (GenericUDFBridge) (genericUDF); UDFType bridgeUDFType = AnnotationUtils.getAnnotation(bridge.getUdfClass(), UDFType.class); if (bridgeUDFType != null && bridgeUDFType.deterministic() == false) { return false; } } if (genericUDF instanceof GenericUDFMacro) { GenericUDFMacro macro = (GenericUDFMacro) (genericUDF); return macro.isDeterministic(); } return true; }
@Override public boolean isDeterministic() { try { UDFType udfType = hiveFunctionWrapper.getUDFClass().getAnnotation(UDFType.class); return udfType != null && udfType.deterministic() && !udfType.stateful(); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } }
/** * Checks if this grouping is like distinct, which means that all non-distinct grouping * columns behave like they were distinct - for example min and max operators. */ public boolean isDistinctLike() { ArrayList<AggregationDesc> aggregators = getAggregators(); for (AggregationDesc ad : aggregators) { if (!ad.getDistinct()) { GenericUDAFEvaluator udafEval = ad.getGenericUDAFEvaluator(); UDFType annot = AnnotationUtils.getAnnotation(udafEval.getClass(), UDFType.class); if (annot == null || !annot.distinctLike()) { return false; } } } return true; }
/** * Both UDF and UDAF functions can imply order for analytical functions * * @param functionName * name of function * @return true if a GenericUDF or GenericUDAF exists for this name and implyOrder is true, false * otherwise. * @throws SemanticException */ public static boolean impliesOrder(String functionName) throws SemanticException { FunctionInfo info = getFunctionInfo(functionName); if (info != null && info.isGenericUDF()) { UDFType type = AnnotationUtils.getAnnotation(info.getGenericUDF().getClass(), UDFType.class); if (type != null) { return type.impliesOrder(); } } WindowFunctionInfo windowInfo = getWindowFunctionInfo(functionName); if (windowInfo != null) { return windowInfo.isImpliesOrder(); } return false; }
@UDFType(deterministic = false) public class GenericUDFAssertTrueOOM extends GenericUDF { private ObjectInspectorConverters.Converter conditionConverter = null;
/** * Returns whether a GenericUDF is deterministic or not. */ public static boolean isDeterministic(GenericUDF genericUDF) { if (isStateful(genericUDF)) { // stateful implies non-deterministic, regardless of whatever // the deterministic annotation declares return false; } UDFType genericUDFType = AnnotationUtils.getAnnotation(genericUDF.getClass(), UDFType.class); if (genericUDFType != null && genericUDFType.deterministic() == false) { return false; } if (genericUDF instanceof GenericUDFBridge) { GenericUDFBridge bridge = (GenericUDFBridge) (genericUDF); UDFType bridgeUDFType = AnnotationUtils.getAnnotation(bridge.getUdfClass(), UDFType.class); if (bridgeUDFType != null && bridgeUDFType.deterministic() == false) { return false; } } if (genericUDF instanceof GenericUDFMacro) { GenericUDFMacro macro = (GenericUDFMacro) (genericUDF); return macro.isDeterministic(); } return true; }
@Override public boolean isDeterministic() { try { UDFType udfType = hiveFunctionWrapper.getUDFClass().getAnnotation(UDFType.class); return udfType != null && udfType.deterministic() && !udfType.stateful(); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } }
/** * Checks if this grouping is like distinct, which means that all non-distinct grouping * columns behave like they were distinct - for example min and max operators. */ public boolean isDistinctLike() { ArrayList<AggregationDesc> aggregators = getAggregators(); for (AggregationDesc ad : aggregators) { if (!ad.getDistinct()) { GenericUDAFEvaluator udafEval = ad.getGenericUDAFEvaluator(); UDFType annot = AnnotationUtils.getAnnotation(udafEval.getClass(), UDFType.class); if (annot == null || !annot.distinctLike()) { return false; } } } return true; }
/** * Both UDF and UDAF functions can imply order for analytical functions * * @param functionName * name of function * @return true if a GenericUDF or GenericUDAF exists for this name and implyOrder is true, false * otherwise. * @throws SemanticException */ public static boolean impliesOrder(String functionName) throws SemanticException { FunctionInfo info = getFunctionInfo(functionName); if (info != null && info.isGenericUDF()) { UDFType type = AnnotationUtils.getAnnotation(info.getGenericUDF().getClass(), UDFType.class); if (type != null) { return type.impliesOrder(); } } WindowFunctionInfo windowInfo = getWindowFunctionInfo(functionName); if (windowInfo != null) { return windowInfo.isImpliesOrder(); } return false; }
@Description( name = "lag", value = "LAG (scalar_expression [,offset] [,default]) OVER ([query_partition_clause] order_by_clause); " + "The LAG function is used to access data from a previous row.", extended = "Example:\n " + "select p1.p_mfgr, p1.p_name, p1.p_size,\n" + " p1.p_size - lag(p1.p_size,1,p1.p_size) over( distribute by p1.p_mfgr sort by p1.p_name) as deltaSz\n" + " from part p1 join part p2 on p1.p_partkey = p2.p_partkey") @UDFType(impliesOrder = true) public class GenericUDFLag extends GenericUDFLeadLag { @Override protected String _getFnName() { return "lag"; } @Override protected int getIndex(int amt) { return pItr.getIndex() - 1 - amt; } @Override protected Object getRow(int amt) throws HiveException { return pItr.lag(amt + 1); } }
@UDFType(deterministic = false) public class GenericUDFSurrogateKey extends GenericUDF { private static final int DEFAULT_WRITE_ID_BITS = 24;
private <C,I> void register(Class<? extends I> clazz, ArrayListMultimap<String,Class<? extends I>> methods) { Description desc = clazz.getAnnotation(Description.class); String[] names; if (desc != null) { names = desc.name().split(","); for (int i=0; i<names.length; i++) { names[i] = names[i].trim(); } }else{ names = new String[]{clazz.getName().replace('.', '_')}; } UDFType type = clazz.getAnnotation(UDFType.class); if (type != null && !type.deterministic()) { nonDeterministicUDFs.add(clazz); } for(int i=0; i<names.length;i++) { methods.put(names[i].toLowerCase(), clazz); } }
udfType = ((GenericUDFBridge) udf).getUdfClass().getAnnotation(UDFType.class); if (udfType.deterministic() == false) { if (udf.getClass().equals(GenericUDFUnixTimeStamp.class) && children != null && children.size() > 0) {
/** * Returns whether a GenericUDF is stateful or not. */ public static boolean isStateful(GenericUDF genericUDF) { UDFType genericUDFType = AnnotationUtils.getAnnotation(genericUDF.getClass(), UDFType.class); if (genericUDFType != null && genericUDFType.stateful()) { return true; } if (genericUDF instanceof GenericUDFBridge) { GenericUDFBridge bridge = (GenericUDFBridge) genericUDF; UDFType bridgeUDFType = AnnotationUtils.getAnnotation(bridge.getUdfClass(), UDFType.class); if (bridgeUDFType != null && bridgeUDFType.stateful()) { return true; } } if (genericUDF instanceof GenericUDFMacro) { GenericUDFMacro macro = (GenericUDFMacro) (genericUDF); return macro.isStateful(); } return false; }
/** * Checks if this grouping is like distinct, which means that all non-distinct grouping * columns behave like they were distinct - for example min and max operators. */ public boolean isDistinctLike() { ArrayList<AggregationDesc> aggregators = getAggregators(); for (AggregationDesc ad : aggregators) { if (!ad.getDistinct()) { GenericUDAFEvaluator udafEval = ad.getGenericUDAFEvaluator(); UDFType annot = AnnotationUtils.getAnnotation(udafEval.getClass(), UDFType.class); if (annot == null || !annot.distinctLike()) { return false; } } } return true; }