ObjectInspector.Category oiCategory = oi.getCategory(); switch (oiCategory) { case PRIMITIVE: case LIST: ListObjectInspector loi = (ListObjectInspector) oi; int listSize = loi.getListLength(val); ObjectInspector listElementOI = loi.getListElementObjectInspector(); for (int idx = 0; idx < listSize; ++idx) { convertedList.add(convertValue(loi.getListElement(val, idx), listElementOI)); case MAP: MapObjectInspector moi = (MapObjectInspector) oi; int mapSize = moi.getMapSize(val); ObjectInspector mapKeyOI = moi.getMapKeyObjectInspector(); ObjectInspector mapValOI = moi.getMapValueObjectInspector(); Map<?, ?> mapCol = moi.getMap(val); for (Object mapKey : mapCol.keySet()) { StructObjectInspector soi = (StructObjectInspector) oi; List<Object> convertedRow = new ArrayList<Object>(); for (StructField structField : soi.getAllStructFieldRefs()) { Object convertedFieldValue = convertValue( soi.getStructFieldData(val, structField), structField.getFieldObjectInspector()); convertedRow.add(convertedFieldValue);
private static List<?> serializeList(Object f, ListObjectInspector loi) throws SerDeException { List l = loi.getList(f); if (l == null) { return null; ObjectInspector eloi = loi.getListElementObjectInspector(); if (eloi.getCategory() == Category.PRIMITIVE) { List<Object> list = new ArrayList<Object>(l.size()); for (int i = 0; i < l.size(); i++) { list.add(((PrimitiveObjectInspector) eloi).getPrimitiveJavaObject(l.get(i))); } else if (eloi.getCategory() == Category.STRUCT) { List<List<?>> list = new ArrayList<List<?>>(l.size()); for (int i = 0; i < l.size(); i++) { } else if (eloi.getCategory() == Category.LIST) { List<List<?>> list = new ArrayList<List<?>>(l.size()); for (int i = 0; i < l.size(); i++) {
@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if(partial == null) { return; } List<DoubleWritable> partialHistogram = (List<DoubleWritable>) loi.getList(partial); DoubleObjectInspector doi = (DoubleObjectInspector)loi.getListElementObjectInspector(); StdAgg myagg = (StdAgg) agg; myagg.histogram.merge(partialHistogram, doi); }
private List getListObject(ListObjectInspector listObjectInspector, Object listObject) { if (listObjectInspector.getListLength(listObject) < 0) { return null; } List<?> objectList = listObjectInspector.getList(listObject); List<?> list = null; ObjectInspector child = listObjectInspector.getListElementObjectInspector(); switch (child.getCategory()) { case PRIMITIVE: final PrimitiveObjectInspector primitiveObjectInspector = (PrimitiveObjectInspector) child; list = objectList.stream() .map(input -> coercePrimitiveObject(primitiveObjectInspector, input)) .collect(Collectors.toList()); break; default: break; } return list; }
protected EWAHCompressedBitmap wordArrayToBitmap(Object b) { ListObjectInspector lloi = (ListObjectInspector) b1OI; int length = lloi.getListLength(b); ArrayList<LongWritable> bitmapArray = new ArrayList<LongWritable>(); for (int i = 0; i < length; i++) { long l = PrimitiveObjectInspectorUtils.getLong( lloi.getListElement(b, i), (PrimitiveObjectInspector) lloi.getListElementObjectInspector()); bitmapArray.add(new LongWritable(l)); } BitmapObjectInput bitmapObjIn = new BitmapObjectInput(bitmapArray); EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); try { bitmap.readExternal(bitmapObjIn); } catch (IOException e) { throw new RuntimeException(e); } return bitmap; }
private boolean serialize(Object obj, ObjectInspector objInspector, int level, ByteStream.Output ss) throws IOException { switch (objInspector.getCategory()) { case PRIMITIVE: LazyUtils.writePrimitiveUTF8(ss, obj, (PrimitiveObjectInspector) objInspector, escaped, escapeChar, needsEscape); char separator = (char) separators[level]; ListObjectInspector loi = (ListObjectInspector) objInspector; List<?> list = loi.getList(obj); ObjectInspector eoi = loi.getListElementObjectInspector(); if (list == null) { return false; char keyValueSeparator = (char) separators[level + 1]; MapObjectInspector moi = (MapObjectInspector) objInspector; ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map<?, ?> map = moi.getMap(obj); if (map == null) { return false; serialize(currentItem, fields.get(i).getFieldObjectInspector(), level + 1, ss); throw new RuntimeException("Unknown category type: " + objInspector.getCategory());
switch (oi.getCategory()) { case PRIMITIVE: { return oi.getClass().getSimpleName(); ListObjectInspector loi = (ListObjectInspector) oi; return oi.getClass().getSimpleName() + "<" + getObjectInspectorName(loi.getListElementObjectInspector()) + ">"; + getObjectInspectorName(moi.getMapKeyObjectInspector()) + "," + getObjectInspectorName(moi.getMapValueObjectInspector()) + ">"; result.append(oi.getClass().getSimpleName() + "<"); StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { result.append(fields.get(i).getFieldName()); result.append(":"); result.append(getObjectInspectorName(fields.get(i) .getFieldObjectInspector())); if (i == fields.size() - 1) { result.append(">");
switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector loi = (PrimitiveObjectInspector) oi; if (objectInspectorOption == ObjectInspectorCopyOption.DEFAULT) { objectInspectorOption = loi.preferWritable() ? ObjectInspectorCopyOption.WRITABLE : ObjectInspectorCopyOption.JAVA; result = loi.getPrimitiveJavaObject(o); if (loi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP) { result = PrimitiveObjectInspectorFactory.javaTimestampObjectInspector.copyObject(result); } else if (loi.getPrimitiveCategory() == int length = loi.getListLength(o); ArrayList<Object> list = new ArrayList<Object>(length); for (int i = 0; i < length; i++) { list.add(copyToStandardObject(loi.getListElement(o, i), loi .getListElementObjectInspector(), objectInspectorOption)); List<? extends StructField> fields = soi.getAllStructFieldRefs(); ArrayList<Object> struct = new ArrayList<Object>(fields.size()); for (StructField f : fields) { struct.add(copyToStandardObject(soi.getStructFieldData(o, f), f .getFieldObjectInspector(), objectInspectorOption));
StructField sf = soi.getStructFieldRef(names[i]); if (sf == null) { throw new MetaException("Invalid Field " + names[i]); } else { oi = sf.getFieldObjectInspector(); oi = loi.getListElementObjectInspector(); } else if (oi instanceof MapObjectInspector && names[i].equalsIgnoreCase("$key$")) { MapObjectInspector moi = (MapObjectInspector) oi; oi = moi.getMapKeyObjectInspector(); } else if (oi instanceof MapObjectInspector && names[i].equalsIgnoreCase("$value$")) { MapObjectInspector moi = (MapObjectInspector) oi; oi = moi.getMapValueObjectInspector(); } else { throw new MetaException("Unknown type for " + names[i]);
switch (oi.getCategory()) { case PRIMITIVE: Writable writable = (Writable) ((PrimitiveObjectInspector) oi).getPrimitiveWritableObject(data); return writableWriter.write(writable, generator); generator.writeBeginArray(); for (int i = 0; i < loi.getListLength(data); i++) { Result result = write(loi.getListElement(data, i), loi.getListElementObjectInspector(), generator); if (!result.isSuccesful()) { return result; for (Map.Entry<?, ?> entry : moi.getMap(data).entrySet()) { Result result = write(entry.getValue(), moi.getMapValueObjectInspector(), generator); if (!result.isSuccesful()) { return result; StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> refs = soi.getAllStructFieldRefs();
if (oi1.getCategory() != oi2.getCategory()) { return oi1.getCategory().compareTo(oi2.getCategory()); PrimitiveObjectInspector poi1 = ((PrimitiveObjectInspector) oi1); PrimitiveObjectInspector poi2 = ((PrimitiveObjectInspector) oi2); if (poi1.getPrimitiveCategory() != poi2.getPrimitiveCategory()) { return poi1.getPrimitiveCategory().compareTo( poi2.getPrimitiveCategory()); ListObjectInspector loi1 = (ListObjectInspector) oi1; ListObjectInspector loi2 = (ListObjectInspector) oi2; int minimum = Math.min(loi1.getListLength(o1), loi2.getListLength(o2)); for (int i = 0; i < minimum; i++) { int r = compare(loi1.getListElement(o1, i), loi1 .getListElementObjectInspector(), loi2.getListElement(o2, i), loi2 .getListElementObjectInspector(), mapEqualComparer, nullValueOpt); if (r != 0) { return loi1.getListLength(o1) - loi2.getListLength(o2);
switch (objInspector.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) objInspector; switch (poi.getPrimitiveCategory()) { case VOID: { return; + poi.getPrimitiveCategory()); ObjectInspector eoi = loi.getListElementObjectInspector(); int size = loi.getListLength(obj); LazyBinaryUtils.writeVInt(byteStream, size); for (int eid = 0; eid < size; eid++) { if (null != loi.getListElement(obj, eid)) { nullByte |= 1 << (eid % 8); serialize(byteStream, loi.getListElement(obj, eid), eoi, false, warnedOnceNullMapKey); ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map<?, ?> map = moi.getMap(obj); if (ObjectInspector.Category.STRUCT.equals(objInspector.getCategory()) ) { + objInspector.getCategory());
@Override public StructObjectInspector initialize(ObjectInspector[] ois) throws UDFArgumentException { //There should be one argument that is a array of struct if (ois.length!=1){ throw new UDFArgumentException("UDF tables only one argument"); } if (ois[0].getCategory()!= Category.LIST){ throw new UDFArgumentException("Top level object must be an array but " + "was "+ois[0].getTypeName()); } li = (ListObjectInspector) ois[0]; ObjectInspector sub=li.getListElementObjectInspector(); if (sub.getCategory() != Category.STRUCT){ throw new UDFArgumentException("The sub element must be struct, but was "+sub.getTypeName()); } return (StructObjectInspector) sub; }
if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " was passed as parameter 1."); switch (((PrimitiveObjectInspector) parameters[0]).getPrimitiveCategory()) { case BYTE: case SHORT: throw new UDFArgumentTypeException(0, "Only numeric type arguments are accepted but " + parameters[0].getTypeName() + " was passed as parameter 1."); if(((ListObjectInspector) parameters[1]).getListElementObjectInspector().getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, verifyFractionType(((ListObjectInspector) parameters[1]).getListElementObjectInspector()); wantManyQuantiles = true; break; + "parameter 3, but " + parameters[2].getTypeName() + " was passed instead."); switch(((PrimitiveObjectInspector) parameters[2]).getPrimitiveCategory()) { case BYTE: case SHORT:
"bigint,c6:float,c7:double,c8:binary,c9:string,c10:struct<" + "c1:int>,c11:map<int,int>,c12:uniontype<int>,c13:array<timestamp>>", inspector.getTypeName()); assertEquals(null, inspector.getAllStructFieldRefs().get(0).getFieldComment()); assertEquals(null, inspector.getStructFieldRef("UNKNOWN")); OrcStruct s1 = new OrcStruct(13); for(int i=0; i < 13; ++i) { assertEquals(list, inspector.getStructFieldsDataAsList(s1)); ListObjectInspector listOI = (ListObjectInspector) inspector.getAllStructFieldRefs().get(12).getFieldObjectInspector(); assertEquals(ObjectInspector.Category.LIST, listOI.getCategory()); assertEquals(10, listOI.getListElement(list, 10)); assertEquals(null, listOI.getListElement(list, -1)); assertEquals(null, listOI.getListElement(list, 13)); assertEquals(13, listOI.getListLength(list)); map.put(3,6); MapObjectInspector mapOI = (MapObjectInspector) inspector.getAllStructFieldRefs().get(10).getFieldObjectInspector(); assertEquals(3, mapOI.getMapSize(map)); assertEquals(4, mapOI.getMapValueElement(map, 2));
switch (oi.getCategory()) { case PRIMITIVE: return true; case LIST: ListObjectInspector loi = (ListObjectInspector) oi; return compareSupported(loi.getListElementObjectInspector()); case STRUCT: StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int f = 0; f < fields.size(); f++) { if (!compareSupported(fields.get(f).getFieldObjectInspector())) { return false;
prepareInSet(arguments); switch (compareOI.getCategory()) { case PRIMITIVE: { if (constantInSet.contains(((PrimitiveObjectInspector) compareOI) .getPrimitiveJavaObject(conversionHelper.convertIfNecessary(arguments[0].get(), argumentOIs[0])))) { bw.set(true); if (constantInSet.contains(((ListObjectInspector) compareOI).getList(conversionHelper .convertIfNecessary(arguments[0].get(), argumentOIs[0])))) { bw.set(true); if (constantInSet.contains(((MapObjectInspector) compareOI).getMap(conversionHelper .convertIfNecessary(arguments[0].get(), argumentOIs[0])))) { bw.set(true); if (constantInSet.contains(((StructObjectInspector) compareOI).getStructFieldsDataAsList(conversionHelper .convertIfNecessary(arguments[0].get(), argumentOIs[0])))) { bw.set(true); + compareOI.getCategory());
@Override public ObjectInspector initialize(ObjectInspector[] arg0) throws UDFArgumentException { if (arg0.length != 2) { throw new UDFArgumentException("array_index takes an array and an int as arguments"); } if (arg0[0].getCategory() != Category.LIST || arg0[1].getCategory() != Category.PRIMITIVE || ((PrimitiveObjectInspector) arg0[1]).getPrimitiveCategory() != PrimitiveCategory.INT) { throw new UDFArgumentException("array_index takes an array and an int as arguments"); } listInspector = (ListObjectInspector) arg0[0]; intInspector = (IntObjectInspector) arg0[1]; return listInspector.getListElementObjectInspector(); }
@Override public StructObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException { if (args.length != 1) { throw new UDFArgumentException("explode() takes only one argument"); } ArrayList<String> fieldNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); switch (args[0].getCategory()) { case LIST: inputOI = args[0]; fieldNames.add("col"); fieldOIs.add(((ListObjectInspector)inputOI).getListElementObjectInspector()); break; case MAP: inputOI = args[0]; fieldNames.add("key"); fieldNames.add("value"); fieldOIs.add(((MapObjectInspector)inputOI).getMapKeyObjectInspector()); fieldOIs.add(((MapObjectInspector)inputOI).getMapValueObjectInspector()); break; default: throw new UDFArgumentException("explode() takes an array or a map as a parameter"); } return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
switch (arguments[0].getCategory()) { case LIST: listObjectInspector = (ListObjectInspector) arguments[0]; throw new UDFArgumentTypeException(0, "Argument 1 of function SORT_ARRAY_BY must be " + serdeConstants.LIST_TYPE_NAME + ", but " + arguments[0].getTypeName() + " was found."); switch (listObjectInspector.getListElementObjectInspector().getCategory()) { case STRUCT: structObjectInspector = (StructObjectInspector) listObjectInspector.getListElementObjectInspector(); break; default: throw new UDFArgumentTypeException(0, "Element[s] of first argument array in function SORT_ARRAY_BY must be " + serdeConstants.STRUCT_TYPE_NAME + ", but " + listObjectInspector.getTypeName() + " was found."); continue; fields[i - 1] = structObjectInspector.getStructFieldRef(getConstantStringValue(arguments, i));