public static boolean isSupportedField(ObjectInspector foi) { if (foi.getCategory() != Category.PRIMITIVE) return false; // not supported PrimitiveCategory pc = ((PrimitiveObjectInspector)foi).getPrimitiveCategory(); if (!SUPPORTED_PRIMITIVES.contains(pc)) return false; // not supported return true; }
private Converter checkTextArguments(ObjectInspector[] arguments, int i) throws UDFArgumentException { if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "Only primitive type arguments are accepted but " + arguments[i].getTypeName() + " is passed."); } Converter converter = ObjectInspectorConverters.getConverter((PrimitiveObjectInspector) arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector); return converter; }
public LazyHCatRecord(Object wrappedObject, ObjectInspector oi) throws Exception { if (oi.getCategory() != Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only make a lazy hcat record from " + "objects of struct types, but we got: " + oi.getTypeName()); } this.soi = (StructObjectInspector)oi; this.wrappedObject = wrappedObject; }
private String describeOi(String desc, ObjectInspector keyOi) { for (StructField field : ((StructObjectInspector)keyOi).getAllStructFieldRefs()) { ObjectInspector oi = field.getFieldObjectInspector(); String cat = oi.getCategory().toString(); if (oi.getCategory() == Category.PRIMITIVE) { cat = ((PrimitiveObjectInspector)oi).getPrimitiveCategory().toString(); } desc += field.getFieldName() + ":" + cat + ", "; } return desc; } }
public static void copyStructToArray(Object o, ObjectInspector oi, ObjectInspectorCopyOption objectInspectorOption, Object[] dest, int offset) throws SerDeException { if (o == null) { return; } if (oi.getCategory() != Category.STRUCT) { throw new SerDeException("Unexpected category " + oi.getCategory()); } StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); ++i) { StructField f = fields.get(i); dest[offset + i] = copyToStandardObject(soi.getStructFieldData(o, f), f .getFieldObjectInspector(), objectInspectorOption); } }
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { if (arguments.length < 2) { throw new UDFArgumentLengthException( "The function CONCAT_WS(separator,[string | array(string)]+) " + "needs at least two arguments."); } // check if argument is a string or an array of strings for (int i = 0; i < arguments.length; i++) { switch(arguments[i].getCategory()) { case LIST: if (isStringOrVoidType( ((ListObjectInspector) arguments[i]).getListElementObjectInspector())) { break; } case PRIMITIVE: if (isStringOrVoidType(arguments[i])) { break; } default: throw new UDFArgumentTypeException(i, "Argument " + (i + 1) + " of function CONCAT_WS must be \"" + serdeConstants.STRING_TYPE_NAME + " or " + serdeConstants.LIST_TYPE_NAME + "<" + serdeConstants.STRING_TYPE_NAME + ">\", but \"" + arguments[i].getTypeName() + "\" was found."); } } argumentOIs = arguments; return PrimitiveObjectInspectorFactory.writableStringObjectInspector; }
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { if (arguments.length != 2) { throw new UDFArgumentLengthException( "InBloomFilter requires exactly 2 arguments but got " + arguments.length); } // Verify individual arguments if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "The 1st argument must be a primitive type but " + arguments[0].getTypeName() + " was passed"); } if (((PrimitiveObjectInspector) arguments[1]).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.BINARY) { throw new UDFArgumentTypeException(1, "The 2nd argument must be a binary type but " + arguments[1].getTypeName() + " was passed"); } valObjectInspector = arguments[0]; bloomFilterObjectInspector = arguments[1]; assert bloomFilterObjectInspector instanceof WritableBinaryObjectInspector; initializedBloomFilter = false; return PrimitiveObjectInspectorFactory.javaBooleanObjectInspector; }
@Override protected Object extractField(Object target) { List<String> flNames = fieldNames; for (int i = 0; i < flNames.size(); i++) { String fl = flNames.get(i); if (target instanceof HiveType) { HiveType type = (HiveType) target; ObjectInspector inspector = type.getObjectInspector(); if (inspector instanceof StructObjectInspector) { StructObjectInspector soi = (StructObjectInspector) inspector; StructField field = soi.getStructFieldRef(fl); ObjectInspector foi = field.getFieldObjectInspector(); Assert.isTrue(foi.getCategory() == ObjectInspector.Category.PRIMITIVE, String.format("Field [%s] needs to be a primitive; found [%s]", fl, foi.getTypeName())); // expecting a writeable - simply do a toString target = soi.getStructFieldData(type.getObject(), field); } else { return FieldExtractor.NOT_FOUND; } } else { return FieldExtractor.NOT_FOUND; } } if (target == null || target instanceof NullWritable) { return StringUtils.EMPTY; } return target.toString(); }
private List getListObject(ListObjectInspector listObjectInspector, Object listObject) { if (listObjectInspector.getListLength(listObject) < 0) { return null; } List<?> objectList = listObjectInspector.getList(listObject); List<?> list = null; ObjectInspector child = listObjectInspector.getListElementObjectInspector(); switch (child.getCategory()) { case PRIMITIVE: final PrimitiveObjectInspector primitiveObjectInspector = (PrimitiveObjectInspector) child; list = objectList.stream() .map(input -> coercePrimitiveObject(primitiveObjectInspector, input)) .collect(Collectors.toList()); break; default: break; } return list; }
public static boolean hasComplexObjects(LazyBinaryStructObjectInspector lazyBinaryStructObjectInspector) { List<? extends StructField> fields = lazyBinaryStructObjectInspector.getAllStructFieldRefs(); for (StructField field : fields) { if (field.getFieldObjectInspector().getCategory() != Category.PRIMITIVE) { return true; } } return false; }
void setupParameterOIs(ObjectInspector[] arguments, int start) throws UDFArgumentTypeException { int length = arguments.length - start; parameterOIs = new PrimitiveObjectInspector[length]; parameterTypes = new PrimitiveTypeEntry[length]; parameterClasses = new Class[length]; parameterJavaValues = new Object[length]; for (int i = 0; i < length; i++) { if (arguments[i + start].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "The parameters of GenericUDFReflect(class,method[,arg1[,arg2]...])" + " must be primitive (int, double, string, etc)."); } parameterOIs[i] = (PrimitiveObjectInspector)arguments[i + start]; parameterTypes[i] = PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveCategory( parameterOIs[i].getPrimitiveCategory()); parameterClasses[i] = parameterTypes[i].primitiveJavaType == null ? parameterTypes[i].primitiveJavaClass : parameterTypes[i].primitiveJavaType; } }
@Override public StructObjectInspector initialize(ObjectInspector[] ois) throws UDFArgumentException { //There should be one argument that is a array of struct if (ois.length!=1){ throw new UDFArgumentException("UDF tables only one argument"); } if (ois[0].getCategory()!= Category.LIST){ throw new UDFArgumentException("Top level object must be an array but " + "was "+ois[0].getTypeName()); } li = (ListObjectInspector) ois[0]; ObjectInspector sub=li.getListElementObjectInspector(); if (sub.getCategory() != Category.STRUCT){ throw new UDFArgumentException("The sub element must be struct, but was "+sub.getTypeName()); } return (StructObjectInspector) sub; }
protected void checkIfPrimitive(ObjectInspector[] arguments, int i, String argOrder) throws UDFArgumentTypeException { ObjectInspector.Category oiCat = arguments[i].getCategory(); if (oiCat != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, getFuncName() + " only takes primitive types as " + argOrder + " argument, got " + oiCat); } }
private String describeOi(String desc, ObjectInspector keyOi) { for (StructField field : ((StructObjectInspector)keyOi).getAllStructFieldRefs()) { ObjectInspector oi = field.getFieldObjectInspector(); String cat = oi.getCategory().toString(); if (oi.getCategory() == Category.PRIMITIVE) { cat = ((PrimitiveObjectInspector)oi).getPrimitiveCategory().toString(); } desc += field.getFieldName() + ":" + cat + ", "; } return desc; } }
private static void unpackStructObject(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj cStatsObj) throws UnsupportedDoubleException { if (oi.getCategory() != ObjectInspector.Category.STRUCT) { throw new RuntimeException("Invalid object datatype : " + oi.getCategory().toString()); } StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); List<Object> list = soi.getStructFieldsDataAsList(o); for (int i = 0; i < fields.size(); i++) { // Get the field objectInspector, fieldName and the field object. ObjectInspector foi = fields.get(i).getFieldObjectInspector(); Object f = (list == null ? null : list.get(i)); String fieldName = fields.get(i).getFieldName(); if (foi.getCategory() == ObjectInspector.Category.PRIMITIVE) { unpackPrimitiveObject(foi, f, fieldName, cStatsObj); } else { unpackStructObject(foi, f, fieldName, cStatsObj); } } } }
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { GenericUDFUtils.ReturnObjectInspectorResolver returnOIResolver; returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true); checkArgsSize(arguments, 1, 1); switch(arguments[0].getCategory()) { case LIST: if(!((ListObjectInspector)(arguments[0])).getListElementObjectInspector() .getCategory().equals(ObjectInspector.Category.UNION)) { break; } default: throw new UDFArgumentTypeException(0, "Argument 1" + " of function SORT_ARRAY must be " + serdeConstants.LIST_TYPE_NAME + ", and element type should be either primitive, list, struct, or map, " + "but " + arguments[0].getTypeName() + " was found."); } ObjectInspector elementObjectInspector = ((ListObjectInspector)(arguments[0])).getListElementObjectInspector(); argumentOIs = arguments; converters = new Converter[arguments.length]; ObjectInspector returnOI = returnOIResolver.get(elementObjectInspector); converters[0] = ObjectInspectorConverters.getConverter(elementObjectInspector, returnOI); return ObjectInspectorFactory.getStandardListObjectInspector(returnOI); }
public static boolean isSupportedField(ObjectInspector foi) { if (foi.getCategory() != Category.PRIMITIVE) return false; // not supported PrimitiveCategory pc = ((PrimitiveObjectInspector)foi).getPrimitiveCategory(); if (!SUPPORTED_PRIMITIVES.contains(pc)) return false; // not supported return true; }
private Converter checkIntArguments(ObjectInspector[] arguments, int i) throws UDFArgumentException { if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "Only primitive type arguments are accepted but " + arguments[i].getTypeName() + " is passed."); } PrimitiveCategory inputType = ((PrimitiveObjectInspector) arguments[i]).getPrimitiveCategory(); Converter converter; switch (inputType) { case INT: case SHORT: case BYTE: converter = ObjectInspectorConverters.getConverter((PrimitiveObjectInspector) arguments[i], PrimitiveObjectInspectorFactory.writableIntObjectInspector); break; default: throw new UDFArgumentTypeException(i + 1, udfName + " only takes INT/SHORT/BYTE types as " + (i + 1) + "-ths argument, got " + inputType); } return converter; } }
private static boolean isInstanceOfSettableOI(ObjectInspector oi) { switch (oi.getCategory()) { case PRIMITIVE: return isInstanceOfSettablePrimitiveOI((PrimitiveObjectInspector)oi); case STRUCT: return oi instanceof SettableStructObjectInspector; case LIST: return oi instanceof SettableListObjectInspector; case MAP: return oi instanceof SettableMapObjectInspector; case UNION: return oi instanceof SettableUnionObjectInspector; default: throw new RuntimeException("Hive internal error inside isAssignableFromSettableOI : " + oi.getTypeName() + " not supported yet."); } }
@Override public byte[] serializeKey(Object object, StructField field) throws IOException { ObjectInspector inspector = field.getFieldObjectInspector(); if (inspector.getCategory() != ObjectInspector.Category.STRUCT) { throw new IllegalStateException("invalid type value " + inspector.getTypeName()); } output.reset(); for (Object element : ((StructObjectInspector)inspector).getStructFieldsDataAsList(object)) { output.write(toBinary(String.valueOf(element).getBytes(), FIXED_LENGTH, false, false)); } return output.getLength() > 0 ? output.toByteArray() : null; }