@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); genericUDTF = conf.getGenericUDTF(); collector = new UDTFCollector(this); genericUDTF.setCollector(collector); udtfInputOI = (StructObjectInspector) inputObjInspectors[0]; objToSendToUDTF = new Object[udtfInputOI.getAllStructFieldRefs().size()]; MapredContext context = MapredContext.get(); if (context != null) { context.setup(genericUDTF); } StructObjectInspector udtfOutputOI = genericUDTF.initialize(udtfInputOI); if (conf.isOuterLV()) { outerObj = Arrays.asList(new Object[udtfOutputOI.getAllStructFieldRefs().size()]); } // Since we're passing the object output by the UDTF directly to the next // operator, we can use the same OI. outputObjInspector = udtfOutputOI; // Set up periodic progress reporting in case the UDTF doesn't output rows // for a while if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEUDTFAUTOPROGRESS)) { autoProgressor = new AutoProgressor(this.getClass().getName(), reporter, Utilities.getDefaultNotificationInterval(hconf), HiveConf.getTimeVar( hconf, HiveConf.ConfVars.HIVES_AUTO_PROGRESS_TIMEOUT, TimeUnit.MILLISECONDS)); autoProgressor.go(); } }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); genericUDTF = conf.getGenericUDTF(); collector = new UDTFCollector(this); genericUDTF.setCollector(collector); udtfInputOI = (StructObjectInspector) inputObjInspectors[0]; objToSendToUDTF = new Object[udtfInputOI.getAllStructFieldRefs().size()]; MapredContext context = MapredContext.get(); if (context != null) { context.setup(genericUDTF); } StructObjectInspector udtfOutputOI = genericUDTF.initialize(udtfInputOI); if (conf.isOuterLV()) { outerObj = Arrays.asList(new Object[udtfOutputOI.getAllStructFieldRefs().size()]); } // Since we're passing the object output by the UDTF directly to the next // operator, we can use the same OI. outputObjInspector = udtfOutputOI; // Set up periodic progress reporting in case the UDTF doesn't output rows // for a while if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEUDTFAUTOPROGRESS)) { autoProgressor = new AutoProgressor(this.getClass().getName(), reporter, Utilities.getDefaultNotificationInterval(hconf), HiveConf.getTimeVar( hconf, HiveConf.ConfVars.HIVES_AUTO_PROGRESS_TIMEOUT, TimeUnit.MILLISECONDS)); autoProgressor.go(); } }
@Override public void open(FunctionContext context) throws Exception { function = hiveFunctionWrapper.createFunction(); function.setCollector(collector); initialized = false; }
@Override public DataType getResultType(Object[] arguments, Class[] argTypes) { try { if (null == function) { function = hiveFunctionWrapper.createFunction(); function.setCollector(collector); collector.setReturnInspector(returnInspector); } List<Boolean> constants = new ArrayList<>(); for (Object argument : arguments) { if (null == argument) { constants.add(false); } else { constants.add(true); } } hiveFunctionWrapper.setConstants(constants); // Initialize at the client side initialize(arguments); } catch (IllegalAccessException | InstantiationException | ClassNotFoundException | UDFArgumentException e) { throw new RuntimeException(e); } return HiveInspectors.inspectorToDataType(returnInspector); }
@Override protected void initializeOp(Configuration hconf) throws HiveException { conf.getGenericUDTF().setCollector(new UDTFCollector(this)); // Make an object inspector [] of the arguments to the UDTF List<? extends StructField> inputFields = ((StandardStructObjectInspector) inputObjInspectors[0]).getAllStructFieldRefs(); udtfInputOIs = new ObjectInspector[inputFields.size()]; for (int i = 0; i < inputFields.size(); i++) { udtfInputOIs[i] = inputFields.get(i).getFieldObjectInspector(); } objToSendToUDTF = new Object[inputFields.size()]; StructObjectInspector udtfOutputOI = conf.getGenericUDTF().initialize( udtfInputOIs); // Since we're passing the object output by the UDTF directly to the next // operator, we can use the same OI. outputObjInspector = udtfOutputOI; // Set up periodic progress reporting in case the UDTF doesn't output rows // for a while if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEUDTFAUTOPROGRESS)) { autoProgressor = new AutoProgressor(this.getClass().getName(), reporter, Utilities.getDefaultNotificationInterval(hconf), HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVES_AUTO_PROGRESS_TIMEOUT) * 1000); autoProgressor.go(); } // Initialize the rest of the operator DAG super.initializeOp(hconf); }
@Override protected Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException { Collection<Future<?>> result = super.initializeOp(hconf); genericUDTF = conf.getGenericUDTF(); collector = new UDTFCollector(this); genericUDTF.setCollector(collector); udtfInputOI = (StructObjectInspector) inputObjInspectors[0]; objToSendToUDTF = new Object[udtfInputOI.getAllStructFieldRefs().size()]; MapredContext context = MapredContext.get(); if (context != null) { context.setup(genericUDTF); } StructObjectInspector udtfOutputOI = genericUDTF.initialize(udtfInputOI); if (conf.isOuterLV()) { outerObj = Arrays.asList(new Object[udtfOutputOI.getAllStructFieldRefs().size()]); } // Since we're passing the object output by the UDTF directly to the next // operator, we can use the same OI. outputObjInspector = udtfOutputOI; // Set up periodic progress reporting in case the UDTF doesn't output rows // for a while if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEUDTFAUTOPROGRESS)) { autoProgressor = new AutoProgressor(this.getClass().getName(), reporter, Utilities.getDefaultNotificationInterval(hconf), HiveConf.getTimeVar( hconf, HiveConf.ConfVars.HIVES_AUTO_PROGRESS_TIMEOUT, TimeUnit.MILLISECONDS)); autoProgressor.go(); } return result; }
@Override public Object exec(Tuple input) throws IOException { if (!inited) { udtf.configure(instantiateMapredContext()); schemaInfo.init(getInputSchema(), udtf, constantsInfo); inited = true; } if (collector == null) { collector = new HiveUDTFCollector(); udtf.setCollector(collector); } else { collector.init(); } try { if (!endOfAllInput) { udtf.process(input.getAll().toArray()); } else { udtf.close(); } } catch (Exception e) { throw new IOException(e); } return collector.getBag(); }