@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); genericUDTF = conf.getGenericUDTF(); collector = new UDTFCollector(this); genericUDTF.setCollector(collector); udtfInputOI = (StructObjectInspector) inputObjInspectors[0]; objToSendToUDTF = new Object[udtfInputOI.getAllStructFieldRefs().size()]; MapredContext context = MapredContext.get(); if (context != null) { context.setup(genericUDTF); } StructObjectInspector udtfOutputOI = genericUDTF.initialize(udtfInputOI); if (conf.isOuterLV()) { outerObj = Arrays.asList(new Object[udtfOutputOI.getAllStructFieldRefs().size()]); } // Since we're passing the object output by the UDTF directly to the next // operator, we can use the same OI. outputObjInspector = udtfOutputOI; // Set up periodic progress reporting in case the UDTF doesn't output rows // for a while if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEUDTFAUTOPROGRESS)) { autoProgressor = new AutoProgressor(this.getClass().getName(), reporter, Utilities.getDefaultNotificationInterval(hconf), HiveConf.getTimeVar( hconf, HiveConf.ConfVars.HIVES_AUTO_PROGRESS_TIMEOUT, TimeUnit.MILLISECONDS)); autoProgressor.go(); } }
@Override public void process(Object row, int tag) throws HiveException { // The UDTF expects arguments in an object[] StructObjectInspector soi = (StructObjectInspector) inputObjInspectors[tag]; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { objToSendToUDTF[i] = soi.getStructFieldData(row, fields.get(i)); } genericUDTF.process(objToSendToUDTF); if (conf.isOuterLV() && collector.getCounter() == 0) { collector.collect(outerObj); } collector.reset(); }
@Override protected void closeOp(boolean abort) throws HiveException { conf.getGenericUDTF().close(); super.closeOp(abort); } }
@Override public Object exec(Tuple input) throws IOException { if (!inited) { udtf.configure(instantiateMapredContext()); schemaInfo.init(getInputSchema(), udtf, constantsInfo); inited = true; } if (collector == null) { collector = new HiveUDTFCollector(); udtf.setCollector(collector); } else { collector.init(); } try { if (!endOfAllInput) { udtf.process(input.getAll().toArray()); } else { udtf.close(); } } catch (Exception e) { throw new IOException(e); } return collector.getBag(); }
public StructObjectInspector initialize(StructObjectInspector argOIs) throws UDFArgumentException { List<? extends StructField> inputFields = argOIs.getAllStructFieldRefs(); ObjectInspector[] udtfInputOIs = new ObjectInspector[inputFields.size()]; for (int i = 0; i < inputFields.size(); i++) { udtfInputOIs[i] = inputFields.get(i).getFieldObjectInspector(); } return initialize(udtfInputOIs); }
@Override public void open(FunctionContext context) throws Exception { function = hiveFunctionWrapper.createFunction(); function.setCollector(collector); initialized = false; }
void setup(GenericUDTF genericUDTF) { if (needConfigure(genericUDTF)) { genericUDTF.configure(this); } // close is called by UDTFOperator }
public StructObjectInspector initialize(StructObjectInspector argOIs) throws UDFArgumentException { List<? extends StructField> inputFields = argOIs.getAllStructFieldRefs(); ObjectInspector[] udtfInputOIs = new ObjectInspector[inputFields.size()]; for (int i = 0; i < inputFields.size(); i++) { udtfInputOIs[i] = inputFields.get(i).getFieldObjectInspector(); } return initialize(udtfInputOIs); }
@Override public DataType getResultType(Object[] arguments, Class[] argTypes) { try { if (null == function) { function = hiveFunctionWrapper.createFunction(); function.setCollector(collector); collector.setReturnInspector(returnInspector); } List<Boolean> constants = new ArrayList<>(); for (Object argument : arguments) { if (null == argument) { constants.add(false); } else { constants.add(true); } } hiveFunctionWrapper.setConstants(constants); // Initialize at the client side initialize(arguments); } catch (IllegalAccessException | InstantiationException | ClassNotFoundException | UDFArgumentException e) { throw new RuntimeException(e); } return HiveInspectors.inspectorToDataType(returnInspector); }
void setup(GenericUDTF genericUDTF) { if (needConfigure(genericUDTF)) { genericUDTF.configure(this); } // close is called by UDTFOperator }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); genericUDTF = conf.getGenericUDTF(); collector = new UDTFCollector(this); genericUDTF.setCollector(collector); udtfInputOI = (StructObjectInspector) inputObjInspectors[0]; objToSendToUDTF = new Object[udtfInputOI.getAllStructFieldRefs().size()]; MapredContext context = MapredContext.get(); if (context != null) { context.setup(genericUDTF); } StructObjectInspector udtfOutputOI = genericUDTF.initialize(udtfInputOI); if (conf.isOuterLV()) { outerObj = Arrays.asList(new Object[udtfOutputOI.getAllStructFieldRefs().size()]); } // Since we're passing the object output by the UDTF directly to the next // operator, we can use the same OI. outputObjInspector = udtfOutputOI; // Set up periodic progress reporting in case the UDTF doesn't output rows // for a while if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEUDTFAUTOPROGRESS)) { autoProgressor = new AutoProgressor(this.getClass().getName(), reporter, Utilities.getDefaultNotificationInterval(hconf), HiveConf.getTimeVar( hconf, HiveConf.ConfVars.HIVES_AUTO_PROGRESS_TIMEOUT, TimeUnit.MILLISECONDS)); autoProgressor.go(); } }
StructObjectInspector outputOI = genericUDTF.initialize(rowOI);
@Override public void process(Object row, int tag) throws HiveException { // The UDTF expects arguments in an object[] StructObjectInspector soi = (StructObjectInspector) inputObjInspectors[tag]; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { objToSendToUDTF[i] = soi.getStructFieldData(row, fields.get(i)); } genericUDTF.process(objToSendToUDTF); if (conf.isOuterLV() && collector.getCounter() == 0) { collector.collect(outerObj); } collector.reset(); }
@Override protected void closeOp(boolean abort) throws HiveException { conf.getGenericUDTF().close(); super.closeOp(abort); } }
void setup(GenericUDTF genericUDTF) { if (needConfigure(genericUDTF)) { genericUDTF.configure(this); } // close is called by UDTFOperator }
@Override protected void initializeOp(Configuration hconf) throws HiveException { conf.getGenericUDTF().setCollector(new UDTFCollector(this)); // Make an object inspector [] of the arguments to the UDTF List<? extends StructField> inputFields = ((StandardStructObjectInspector) inputObjInspectors[0]).getAllStructFieldRefs(); udtfInputOIs = new ObjectInspector[inputFields.size()]; for (int i = 0; i < inputFields.size(); i++) { udtfInputOIs[i] = inputFields.get(i).getFieldObjectInspector(); } objToSendToUDTF = new Object[inputFields.size()]; StructObjectInspector udtfOutputOI = conf.getGenericUDTF().initialize( udtfInputOIs); // Since we're passing the object output by the UDTF directly to the next // operator, we can use the same OI. outputObjInspector = udtfOutputOI; // Set up periodic progress reporting in case the UDTF doesn't output rows // for a while if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEUDTFAUTOPROGRESS)) { autoProgressor = new AutoProgressor(this.getClass().getName(), reporter, Utilities.getDefaultNotificationInterval(hconf), HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVES_AUTO_PROGRESS_TIMEOUT) * 1000); autoProgressor.go(); } // Initialize the rest of the operator DAG super.initializeOp(hconf); }
StructObjectInspector outputOI = genericUDTF.initialize(rowOI);
public void eval(Object... args) throws HiveException { if (!initialized) { // Initialize at the server side initialize(args); } function.process(args); collector.collectRows(this); }
@Override protected void closeOp(boolean abort) throws HiveException { conf.getGenericUDTF().close(); } }
@Override protected Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException { Collection<Future<?>> result = super.initializeOp(hconf); genericUDTF = conf.getGenericUDTF(); collector = new UDTFCollector(this); genericUDTF.setCollector(collector); udtfInputOI = (StructObjectInspector) inputObjInspectors[0]; objToSendToUDTF = new Object[udtfInputOI.getAllStructFieldRefs().size()]; MapredContext context = MapredContext.get(); if (context != null) { context.setup(genericUDTF); } StructObjectInspector udtfOutputOI = genericUDTF.initialize(udtfInputOI); if (conf.isOuterLV()) { outerObj = Arrays.asList(new Object[udtfOutputOI.getAllStructFieldRefs().size()]); } // Since we're passing the object output by the UDTF directly to the next // operator, we can use the same OI. outputObjInspector = udtfOutputOI; // Set up periodic progress reporting in case the UDTF doesn't output rows // for a while if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEUDTFAUTOPROGRESS)) { autoProgressor = new AutoProgressor(this.getClass().getName(), reporter, Utilities.getDefaultNotificationInterval(hconf), HiveConf.getTimeVar( hconf, HiveConf.ConfVars.HIVES_AUTO_PROGRESS_TIMEOUT, TimeUnit.MILLISECONDS)); autoProgressor.go(); } return result; }