switch (oi.getCategory()) { case PRIMITIVE: { return oi.getClass().getSimpleName(); ListObjectInspector loi = (ListObjectInspector) oi; return oi.getClass().getSimpleName() + "<" + getObjectInspectorName(loi.getListElementObjectInspector()) + ">"; + getObjectInspectorName(moi.getMapKeyObjectInspector()) + "," + getObjectInspectorName(moi.getMapValueObjectInspector()) + ">"; result.append(oi.getClass().getSimpleName() + "<"); StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { result.append(fields.get(i).getFieldName()); result.append(":"); result.append(getObjectInspectorName(fields.get(i) .getFieldObjectInspector())); if (i == fields.size() - 1) { result.append(">"); result.append(oi.getClass().getSimpleName() + "<"); UnionObjectInspector uoi = (UnionObjectInspector)oi; List<ObjectInspector> ois = uoi.getObjectInspectors(); for (int i = 0; i < ois.size(); i++) { if (i > 0) {
switch (oi.getCategory()) { case PRIMITIVE: { if (o == null) { ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector(); ObjectInspector mapValueObjectInspector = moi .getMapValueObjectInspector(); Map<?, ?> omap = moi.getMap(o); if (omap == null) { return true; List<? extends StructField> structFields = soi.getAllStructFieldRefs(); if (o == null) { return true; if (hasAnyNullObject(soi.getStructFieldData(o, structFields.get(i)), structFields.get(i).getFieldObjectInspector())) { return true; } else { if (uoi.getObjectInspectors().size() == 0) { return false; return hasAnyNullObject(uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)));
ObjectInspector.Category oiCategory = oi.getCategory(); switch (oiCategory) { case PRIMITIVE: case LIST: ListObjectInspector loi = (ListObjectInspector) oi; int listSize = loi.getListLength(val); ObjectInspector listElementOI = loi.getListElementObjectInspector(); for (int idx = 0; idx < listSize; ++idx) { convertedList.add(convertValue(loi.getListElement(val, idx), listElementOI)); case MAP: MapObjectInspector moi = (MapObjectInspector) oi; int mapSize = moi.getMapSize(val); ObjectInspector mapKeyOI = moi.getMapKeyObjectInspector(); ObjectInspector mapValOI = moi.getMapValueObjectInspector(); Map<?, ?> mapCol = moi.getMap(val); for (Object mapKey : mapCol.keySet()) { Object convertedMapKey = convertValue(mapKey, mapKeyOI);
@Override public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2) { int mapsize1 = moi1.getMapSize(o1); int mapsize2 = moi2.getMapSize(o2); if (mapsize1 != mapsize2) { return mapsize1 - mapsize2; } ObjectInspector mvoi1 = moi1.getMapValueObjectInspector(); ObjectInspector mvoi2 = moi2.getMapValueObjectInspector(); Map<?, ?> map1 = moi1.getMap(o1); for (Object mk1: map1.keySet()) { int rc = ObjectInspectorUtils.compare(moi1.getMapValueElement(o1, mk1), mvoi1, moi2.getMapValueElement(o2, mk1), mvoi2, this); if (rc != 0) { return rc; } } return 0; } }
private void getMapObject(String parentName, MapObjectInspector mapObjectInspector, Object mapObject, Map<String, Object> parsedMap) { if (mapObjectInspector.getMapSize(mapObject) < 0) { return; } String mapChildFieldNameFormat = StringUtils.replace( StringUtils.format(mapParentFieldNameFormat, parentName), MAP_CHILD_TAG, "%s" ); Map objectMap = mapObjectInspector.getMap(mapObject); PrimitiveObjectInspector key = (PrimitiveObjectInspector) mapObjectInspector.getMapKeyObjectInspector(); PrimitiveObjectInspector value = (PrimitiveObjectInspector) mapObjectInspector.getMapValueObjectInspector(); objectMap.forEach((k, v) -> { String resolvedFieldName = StringUtils.format(mapChildFieldNameFormat, key.getPrimitiveJavaObject(k).toString()); parsedMap.put(resolvedFieldName, value.getPrimitiveJavaObject(v)); }); }
/** * Helper method to return underlying Java Map from * an object-representation that is readable by a provided * MapObjectInspector */ private static Map<?, ?> serializeMap(Object f, MapObjectInspector moi) throws SerDeException { ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map<Object, Object> m = new LinkedHashMap<Object, Object>(); Map<?, ?> readMap = moi.getMap(f); if (readMap == null) { return null; } else { for (Map.Entry<?, ?> entry : readMap.entrySet()) { m.put(serializeField(entry.getKey(), koi), serializeField(entry.getValue(), voi)); } } return m; }
switch (objIns.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) objIns); switch (poi.getPrimitiveCategory()) { case VOID: return 0; + poi.getPrimitiveCategory()); int r = 0; ListObjectInspector listOI = (ListObjectInspector)objIns; ObjectInspector elemOI = listOI.getListElementObjectInspector(); for (int ii = 0; ii < listOI.getListLength(o); ++ii) { r = 31 * r + hashCodeMurmur(listOI.getListElement(o, ii), elemOI, byteBuffer); int r = 0; MapObjectInspector mapOI = (MapObjectInspector)objIns; ObjectInspector keyOI = mapOI.getMapKeyObjectInspector(); ObjectInspector valueOI = mapOI.getMapValueObjectInspector(); Map<?, ?> map = mapOI.getMap(o); for (Map.Entry<?,?> entry : map.entrySet()) { r += hashCodeMurmur(entry.getKey(), keyOI, byteBuffer) ^ byte tag = uOI.getTag(o); return hashCodeMurmur(uOI.getField(o), uOI.getObjectInspectors().get(tag), byteBuffer); throw new RuntimeException("Unknown type: "+ objIns.getTypeName());
result = loi.getPrimitiveJavaObject(o); if (loi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP) { int length = loi.getListLength(o); ArrayList<Object> list = new ArrayList<Object>(length); for (int i = 0; i < length; i++) { list.add(copyToStandardObject(loi.getListElement(o, i), loi .getListElementObjectInspector(), objectInspectorOption)); MapObjectInspector moi = (MapObjectInspector) oi; Map<Object, Object> map = new LinkedHashMap<Object, Object>(); Map<? extends Object, ? extends Object> omap = moi.getMap(o); for (Map.Entry<? extends Object, ? extends Object> entry : omap .entrySet()) { map.put(copyToStandardObject(entry.getKey(), moi .getMapKeyObjectInspector(), objectInspectorOption), copyToStandardObject(entry.getValue(), moi .getMapValueObjectInspector(), objectInspectorOption)); List<? extends StructField> fields = soi.getAllStructFieldRefs(); ArrayList<Object> struct = new ArrayList<Object>(fields.size()); for (StructField f : fields) { struct.add(copyToStandardObject(soi.getStructFieldData(o, f), f .getFieldObjectInspector(), objectInspectorOption));
private boolean serialize(Object obj, ObjectInspector objInspector, int level, ByteStream.Output ss) throws IOException { switch (objInspector.getCategory()) { case PRIMITIVE: LazyUtils.writePrimitiveUTF8(ss, obj, (PrimitiveObjectInspector) objInspector, escaped, escapeChar, needsEscape); char separator = (char) separators[level]; ListObjectInspector loi = (ListObjectInspector) objInspector; List<?> list = loi.getList(obj); ObjectInspector eoi = loi.getListElementObjectInspector(); if (list == null) { return false; char keyValueSeparator = (char) separators[level + 1]; MapObjectInspector moi = (MapObjectInspector) objInspector; ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map<?, ?> map = moi.getMap(obj); if (map == null) { return false; serialize(currentItem, fields.get(i).getFieldObjectInspector(), level + 1, ss); throw new RuntimeException("Unknown category type: " + objInspector.getCategory());
StructField sf = soi.getStructFieldRef(names[i]); if (sf == null) { throw new MetaException("Invalid Field " + names[i]); } else { oi = sf.getFieldObjectInspector(); oi = loi.getListElementObjectInspector(); } else if (oi instanceof MapObjectInspector && names[i].equalsIgnoreCase("$key$")) { MapObjectInspector moi = (MapObjectInspector) oi; oi = moi.getMapKeyObjectInspector(); } else if (oi instanceof MapObjectInspector && names[i].equalsIgnoreCase("$value$")) { MapObjectInspector moi = (MapObjectInspector) oi; oi = moi.getMapValueObjectInspector(); } else { throw new MetaException("Unknown type for " + names[i]);
@Override public StructObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException { if (args.length != 1) { throw new UDFArgumentException("explode() takes only one argument"); } ArrayList<String> fieldNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); switch (args[0].getCategory()) { case LIST: inputOI = args[0]; fieldNames.add("col"); fieldOIs.add(((ListObjectInspector)inputOI).getListElementObjectInspector()); break; case MAP: inputOI = args[0]; fieldNames.add("key"); fieldNames.add("value"); fieldOIs.add(((MapObjectInspector)inputOI).getMapKeyObjectInspector()); fieldOIs.add(((MapObjectInspector)inputOI).getMapValueObjectInspector()); break; default: throw new UDFArgumentException("explode() takes an array or a map as a parameter"); } return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
switch (oi.getCategory()) { case PRIMITIVE: Writable writable = (Writable) ((PrimitiveObjectInspector) oi).getPrimitiveWritableObject(data); return writableWriter.write(writable, generator); generator.writeBeginArray(); for (int i = 0; i < loi.getListLength(data); i++) { Result result = write(loi.getListElement(data, i), loi.getListElementObjectInspector(), generator); if (!result.isSuccesful()) { return result; for (Map.Entry<?, ?> entry : moi.getMap(data).entrySet()) { Result result = write(entry.getValue(), moi.getMapValueObjectInspector(), generator); if (!result.isSuccesful()) { return result; StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> refs = soi.getAllStructFieldRefs();
ObjectInspector listElementOI = ((ListObjectInspector)inspector).getListElementObjectInspector(); if (listElementOI instanceof PrimitiveObjectInspector) { return toListConstDesc(colInfo, inspector, listElementOI); ObjectInspector keyOI = ((MapObjectInspector)inspector).getMapKeyObjectInspector(); ObjectInspector valueOI = ((MapObjectInspector)inspector).getMapValueObjectInspector(); if (keyOI instanceof PrimitiveObjectInspector && valueOI instanceof PrimitiveObjectInspector) { return toMapConstDesc(colInfo, inspector, keyOI, valueOI); List<? extends StructField> fields = ((StructObjectInspector)inspector).getAllStructFieldRefs(); for (StructField field : fields) { allPrimitive &= field.getFieldObjectInspector() instanceof PrimitiveObjectInspector;
@Override public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException, IllegalAccessException { DynamicSerDeTypeBase keyType = getKeyType(); DynamicSerDeTypeBase valueType = getValueType(); WriteNullsProtocol nullProtocol = (oprot instanceof WriteNullsProtocol) ? (WriteNullsProtocol) oprot : null; assert (oi.getCategory() == ObjectInspector.Category.MAP); MapObjectInspector moi = (MapObjectInspector) oi; ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map<?, ?> map = moi.getMap(o); serializeMap = new TMap(keyType.getType(), valueType.getType(), map.size()); oprot.writeMapBegin(serializeMap); for (Object element : map.entrySet()) { Map.Entry it = (Map.Entry) element; Object key = it.getKey(); Object value = it.getValue(); keyType.serialize(key, koi, oprot); if (value == null) { assert (nullProtocol != null); nullProtocol.writeNull(); } else { valueType.serialize(value, voi, oprot); } } // in theory, the below call isn't needed in non thrift_mode, but let's not // get too crazy oprot.writeMapEnd(); }
switch (objInspector.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) objInspector; switch (poi.getPrimitiveCategory()) { case VOID: { return; + poi.getPrimitiveCategory()); ObjectInspector eoi = loi.getListElementObjectInspector(); int size = loi.getListLength(obj); LazyBinaryUtils.writeVInt(byteStream, size); for (int eid = 0; eid < size; eid++) { if (null != loi.getListElement(obj, eid)) { nullByte |= 1 << (eid % 8); ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map<?, ?> map = moi.getMap(obj); if (ObjectInspector.Category.STRUCT.equals(objInspector.getCategory()) ) { + objInspector.getCategory());
"bigint,c6:float,c7:double,c8:binary,c9:string,c10:struct<" + "c1:int>,c11:map<int,int>,c12:uniontype<int>,c13:array<timestamp>>", inspector.getTypeName()); assertEquals(null, inspector.getAllStructFieldRefs().get(0).getFieldComment()); assertEquals(null, inspector.getStructFieldRef("UNKNOWN")); OrcStruct s1 = new OrcStruct(13); for(int i=0; i < 13; ++i) { assertEquals(list, inspector.getStructFieldsDataAsList(s1)); ListObjectInspector listOI = (ListObjectInspector) inspector.getAllStructFieldRefs().get(12).getFieldObjectInspector(); assertEquals(ObjectInspector.Category.LIST, listOI.getCategory()); assertEquals(10, listOI.getListElement(list, 10)); assertEquals(null, listOI.getListElement(list, -1)); assertEquals(null, listOI.getListElement(list, 13)); assertEquals(13, listOI.getListLength(list)); map.put(3,6); MapObjectInspector mapOI = (MapObjectInspector) inspector.getAllStructFieldRefs().get(10).getFieldObjectInspector(); assertEquals(3, mapOI.getMapSize(map)); assertEquals(4, mapOI.getMapValueElement(map, 2));
@Override public Object evaluate(DeferredObject[] arguments) throws HiveException { Object data = arguments[0].get(); if (returnOI.getCategory() == Category.MAP) { result.set(((MapObjectInspector) returnOI).getMapSize(data)); } else if (returnOI.getCategory() == Category.LIST) { result.set(((ListObjectInspector) returnOI).getListLength(data)); } else if (returnOI.getTypeName().equals(serdeConstants.VOID_TYPE_NAME)) { // null result.set(-1); } return result; }
prepareInSet(arguments); switch (compareOI.getCategory()) { case PRIMITIVE: { if (constantInSet.contains(((PrimitiveObjectInspector) compareOI) .getPrimitiveJavaObject(conversionHelper.convertIfNecessary(arguments[0].get(), argumentOIs[0])))) { bw.set(true); if (constantInSet.contains(((ListObjectInspector) compareOI).getList(conversionHelper .convertIfNecessary(arguments[0].get(), argumentOIs[0])))) { bw.set(true); if (constantInSet.contains(((MapObjectInspector) compareOI).getMap(conversionHelper .convertIfNecessary(arguments[0].get(), argumentOIs[0])))) { bw.set(true); if (constantInSet.contains(((StructObjectInspector) compareOI).getStructFieldsDataAsList(conversionHelper .convertIfNecessary(arguments[0].get(), argumentOIs[0])))) { bw.set(true); + compareOI.getCategory());
@Test public void testDeSerializeMap() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"country\":\"Switzerland\",\"languages\":\"Italian\",\"religions\": { \"f\": \"v\", \"n\":null} }"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); StructField sfr = soi.getStructFieldRef("religions"); assertEquals(sfr.getFieldObjectInspector().getCategory(),ObjectInspector.Category.MAP); MapObjectInspector moi = (MapObjectInspector) sfr.getFieldObjectInspector(); Object val = soi.getStructFieldData(result, sfr) ; assertEquals(2, moi.getMapSize(val)); assertEquals("v", moi.getMapValueElement(val, "f")); ObjectInspector voi = moi.getMapValueObjectInspector(); assertTrue(voi instanceof StringObjectInspector); StringObjectInspector svoi = (StringObjectInspector) voi; assertNull( svoi.getPrimitiveJavaObject(moi.getMapValueElement(val,"n"))); }
throws UDFArgumentException { ObjectInspector first = arg0[0]; if (first.getCategory() == Category.MAP) { mapInspector = (MapObjectInspector) first; } else { if (second.getCategory() == Category.LIST) { keyListInspector = (ListObjectInspector) second; } else { if (!(keyListInspector.getListElementObjectInspector().getCategory() == Category.PRIMITIVE)) { throw new UDFArgumentException(" Expecting a primitive as key list elements."); ObjectInspector mapKeyInspector = mapInspector.getMapKeyObjectInspector(); if (!(mapKeyInspector.getCategory() == Category.PRIMITIVE)) { throw new UDFArgumentException(" Expecting a primitive as map key elements."); LOG.info(" Map has key type " + mapKeyInspector.getTypeName()); LOG.info(" Key list has key type " + keyListInspector.getTypeName()); if (((PrimitiveObjectInspector) keyListInspector.getListElementObjectInspector()).getPrimitiveCategory() != ((PrimitiveObjectInspector) mapKeyInspector).getPrimitiveCategory()) { throw new UDFArgumentException(" Expecting keys to be of same types.");