@Override public String getName() { return funcDesc.getFunctionName(); }
public PythonScriptEngine(FunctionDesc functionDesc, boolean firstPhase, boolean lastPhase) { if (!functionDesc.getInvocation().hasPython() && !functionDesc.getInvocation().hasPythonAggregation()) { throw new IllegalStateException("Function type must be 'python'"); } functionSignature = functionDesc.getSignature(); invocationDesc = functionDesc.getInvocation().getPython(); this.firstPhase = firstPhase; this.lastPhase = lastPhase; setSchema(); }
/** * Checks duplicates between pre-loaded functions and UDFs. And they are meged to funcList. * * @param udfs UDF list * @param funcSet set for pre-loaded functions to match signature * @param funcList list to be merged * @throws AmbiguousFunctionException */ private static void checkUDFduplicateAndMerge(List<FunctionDesc> udfs, HashMap<Integer, FunctionDesc> funcSet, List<FunctionDesc> funcList) throws AmbiguousFunctionException { for (FunctionDesc desc: udfs) { // check if (funcSet.containsKey(desc.hashCodeWithoutType())) { throw new AmbiguousFunctionException(String.format("UDF %s", desc.toString())); } // merge funcSet.put(desc.hashCodeWithoutType(), desc); funcList.add(desc); } }
@Override public void init(FunctionInvokeContext context) throws IOException { UDFInvocationDesc udfDesc = functionDesc.getInvocation().getUDF(); URL [] urls = new URL [] { new URL(udfDesc.getPath()) }; URLClassLoader loader = new URLClassLoader(urls); try { Class<?> udfclass = loader.loadClass(udfDesc.getName()); evalMethod = getEvaluateMethod(functionDesc.getParamTypes(), udfclass); } catch (ClassNotFoundException e) { throw new TajoInternalError(e); } }
private static Collection<FunctionDesc> buildFunctionDescs(ScalarFunction annotation, Method method) { List<FunctionDesc> functionDescs = Lists.newArrayList(); FunctionInvocation invocation = new FunctionInvocation(); invocation.setScalar(extractStaticMethodInvocation(method)); FunctionSupplement supplement = extractSupplement(annotation); // primary name functionDescs.add(new FunctionDesc(extractSignature(annotation, null), invocation, supplement)); // for multiple aliases for (String alias : annotation.synonyms()) { functionDescs.add(new FunctionDesc(extractSignature(annotation, alias), invocation, supplement)); } return functionDescs; }
@Override public FunctionListResponse getFunctionList(RpcController controller, SessionedStringProto request) throws ServiceException { try { context.getSessionManager().touch(request.getSessionId().getId()); String functionName = request.getValue(); Collection<FunctionDesc> functions = catalog.getFunctions(); List<CatalogProtos.FunctionDescProto> functionProtos = new ArrayList<CatalogProtos.FunctionDescProto>(); for (FunctionDesc eachFunction: functions) { if (functionName == null || functionName.isEmpty()) { functionProtos.add(eachFunction.getProto()); } else { if(functionName.equals(eachFunction.getFunctionName())) { functionProtos.add(eachFunction.getProto()); } } } return FunctionListResponse.newBuilder() .setState(OK) .addAllFunction(functionProtos) .build(); } catch (Throwable t) { printStackTraceIfError(LOG, t); return FunctionListResponse.newBuilder(). setState(returnError(t)) .build(); } }
FunctionDesc functionDesc = new FunctionDesc(eachFunctionName, function.getClass(), function.getFunctionType(), CatalogUtil.newSimpleDataType(returnType), paramTypes.length == 0 ? CatalogUtil.newSimpleDataTypeArray() : CatalogUtil.newSimpleDataTypeArray(paramTypes)); functionDesc.setDescription(description); functionDesc.setExample(example); functionDesc.setDetail(detail); sqlFuncs.add(functionDesc);
funcDesc = new FunctionDesc(funcProto.getFuncion()); if (type == EvalType.FUNCTION) { current = new GeneralFunctionEval(context, funcDesc, params); if (evalContext != null && funcDesc.getInvocation().hasPython()) { evalContext.addScriptEngine(current, new PythonScriptEngine(funcDesc)); if (type == EvalType.AGG_FUNCTION) { AggregationFunctionCallEval aggFunc = new AggregationFunctionCallEval(new FunctionDesc(funcProto.getFuncion()), params); if (evalContext != null && funcDesc.getInvocation().hasPythonAggregation()) { evalContext.addScriptEngine(current, new PythonScriptEngine(funcDesc, aggFunc.isFirstPhase(), aggFunc.isLastPhase())); new WindowFunctionEval(new FunctionDesc(funcProto.getFuncion()), params, convertWindowFrame(windowFuncProto.getWindowFrame()));
public static boolean isScriptFunction(FunctionDesc desc) { return desc.getInvocation().hasPython(); } }
@Test public void testAnalyzeUDFclass() { Set<Class<? extends UDF>> funcSet = new HashSet<>(); funcSet.add(HiveUDFtest.class); List<FunctionDesc> funcList = new LinkedList<>(); HiveFunctionLoader.buildFunctionsFromUDF(funcSet, funcList, null); assertEquals(funcList.size(), 1); FunctionDesc desc = funcList.get(0); assertEquals("multiplestr", desc.getFunctionName()); assertEquals(false, desc.isDeterministic()); assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); assertEquals(TajoDataTypes.Type.INT4, desc.getParamTypes()[1].getType()); } }
boolean constantOfAllDescendents = true; if (NON_CONSTANT_FUNC_NAMES.contains(evalNode.getFuncDesc().getFunctionName())) { constantOfAllDescendents = false; } else { evalContext.setTimeZone(TimeZone.getTimeZone(timezoneId)); if (evalNode.getFuncDesc().getInvocation().hasPython()) { TajoScriptEngine executor = new PythonScriptEngine(evalNode.getFuncDesc()); try {
if (CatalogUtil.checkIfVariableLengthParamDefinition(TUtil.newList(funcDesc.getParamTypes()))) { DataType lastDataType = funcDesc.getParamTypes()[0]; for (int i = 0; i < givenArgs.length; i++) { if (i < (funcDesc.getParamTypes().length - 1)) { // variable length lastDataType = funcDesc.getParamTypes()[i]; } else { lastDataType = CatalogUtil.newSimpleDataType(CatalogUtil.getPrimitiveTypeOf(lastDataType.getType())); assertEval(funcDesc.getParamTypes().length == givenArgs.length, "The number of parameters is mismatched to the function definition: " + funcDesc.toString()); givenArgs[i] = convertType(ctx, givenArgs[i], funcDesc.getParamTypes()[i]); FunctionType functionType = funcDesc.getFuncType(); if (functionType == FunctionType.GENERAL || functionType == FunctionType.UDF) { } else if (functionType == FunctionType.DISTINCT_AGGREGATION || functionType == FunctionType.DISTINCT_UDA) { throw new UnsupportedException(funcDesc.toString()); } else { throw new UnsupportedException("function type '" + functionType.name() + "'");
@Override public DataType getValueType() { return funcDesc.getReturnType(); }
public boolean equalsSignature(Object obj) { if (obj instanceof FunctionDesc) { FunctionSignature targetSig = ((FunctionDesc)obj).getSignature(); return this.getSignature().equalsWithoutType(targetSig); } return false; }
@Override public EvalNode visitFuncCall(EvalCodeGenContext context, FunctionEval function, Stack<EvalNode> stack) { super.visitFuncCall(context, function, stack); if (!context.symbols.containsKey(function)) { String fieldName = function.getFuncDesc().getFunctionName() + "_" + context.seqId++; context.symbols.put(function, fieldName); context.classWriter.visitField(Opcodes.ACC_PRIVATE, fieldName, "L" + TajoGeneratorAdapter.getInternalName(function.getFuncDesc().getLegacyFuncClass()) + ";", null, null); } return function; } }
assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); assertEquals(1, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); assertEquals("to uppercase", desc.getDescription()); assertEquals(TajoDataTypes.Type.FLOAT8, desc.getReturnType().getType()); assertEquals(2, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.INT4, desc.getParamTypes()[0].getType()); assertEquals(TajoDataTypes.Type.INT4, desc.getParamTypes()[1].getType()); assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); assertEquals(1, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); assertEquals("to uppercase", desc.getDescription()); assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); assertEquals(1, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); assertEquals(2, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[1].getType()); CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT), CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT4), CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT4)); assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); assertEquals(3, desc.getParamTypes().length);
/** * @return Function Instance */ public Function newInstance() { try { Constructor<? extends Function> cons = getLegacyFuncClass().getConstructor(); return cons.newInstance(); } catch (Exception ioe) { throw new TajoInternalError("Cannot initiate function " + signature); } }
public boolean isDistinct() { return funcDesc.getFuncType() == DISTINCT_AGGREGATION || funcDesc.getFuncType() == DISTINCT_UDA; }
public HiveFunctionInvoke(FunctionDesc desc) { super(desc); params = new Writable[desc.getParamTypes().length]; }