@Override public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2) { int mapsize1 = moi1.getMapSize(o1); int mapsize2 = moi2.getMapSize(o2); if (mapsize1 != mapsize2) { return mapsize1 - mapsize2; } ObjectInspector mvoi1 = moi1.getMapValueObjectInspector(); ObjectInspector mvoi2 = moi2.getMapValueObjectInspector(); Map<?, ?> map1 = moi1.getMap(o1); for (Object mk1: map1.keySet()) { int rc = ObjectInspectorUtils.compare(moi1.getMapValueElement(o1, mk1), mvoi1, moi2.getMapValueElement(o2, mk1), mvoi2, this); if (rc != 0) { return rc; } } return 0; } }
private void getMapObject(String parentName, MapObjectInspector mapObjectInspector, Object mapObject, Map<String, Object> parsedMap) { if (mapObjectInspector.getMapSize(mapObject) < 0) { return; } String mapChildFieldNameFormat = StringUtils.replace( StringUtils.format(mapParentFieldNameFormat, parentName), MAP_CHILD_TAG, "%s" ); Map objectMap = mapObjectInspector.getMap(mapObject); PrimitiveObjectInspector key = (PrimitiveObjectInspector) mapObjectInspector.getMapKeyObjectInspector(); PrimitiveObjectInspector value = (PrimitiveObjectInspector) mapObjectInspector.getMapValueObjectInspector(); objectMap.forEach((k, v) -> { String resolvedFieldName = StringUtils.format(mapChildFieldNameFormat, key.getPrimitiveJavaObject(k).toString()); parsedMap.put(resolvedFieldName, value.getPrimitiveJavaObject(v)); }); }
@Override public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2) { int mapsize1 = moi1.getMapSize(o1); int mapsize2 = moi2.getMapSize(o2); if (mapsize1 != mapsize2) { return mapsize1 - mapsize2;
@Override public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2) { int mapsize1 = moi1.getMapSize(o1); int mapsize2 = moi2.getMapSize(o2); if (mapsize1 != mapsize2) { return mapsize1 - mapsize2;
private Object serializeMap(MapTypeInfo typeInfo, MapObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException { // Avro only allows maps with string keys if(!mapHasStringKey(fieldOI.getMapKeyObjectInspector())) { throw new AvroSerdeException("Avro only supports maps with keys as Strings. Current Map is: " + typeInfo.toString()); } ObjectInspector mapKeyObjectInspector = fieldOI.getMapKeyObjectInspector(); ObjectInspector mapValueObjectInspector = fieldOI.getMapValueObjectInspector(); TypeInfo mapKeyTypeInfo = typeInfo.getMapKeyTypeInfo(); TypeInfo mapValueTypeInfo = typeInfo.getMapValueTypeInfo(); Map<?,?> map = fieldOI.getMap(structFieldData); Schema valueType = schema.getValueType(); Map<Object, Object> deserialized = new LinkedHashMap<Object, Object>(fieldOI.getMapSize(structFieldData)); for (Map.Entry<?, ?> entry : map.entrySet()) { deserialized.put(serialize(mapKeyTypeInfo, mapKeyObjectInspector, entry.getKey(), STRING_SCHEMA), serialize(mapValueTypeInfo, mapValueObjectInspector, entry.getValue(), valueType)); } return deserialized; }
@Override public Object evaluate(DeferredObject[] arguments) throws HiveException { Object data = arguments[0].get(); if (returnOI.getCategory() == Category.MAP) { result.set(((MapObjectInspector) returnOI).getMapSize(data)); } else if (returnOI.getCategory() == Category.LIST) { result.set(((ListObjectInspector) returnOI).getListLength(data)); } else if (returnOI.getTypeName().equals(serdeConstants.VOID_TYPE_NAME)) { // null result.set(-1); } return result; }
@Override public Object evaluate(DeferredObject[] arguments) throws HiveException { Object data = arguments[0].get(); if (returnOI.getCategory() == Category.MAP) { result.set(((MapObjectInspector) returnOI).getMapSize(data)); } else if (returnOI.getCategory() == Category.LIST) { result.set(((ListObjectInspector) returnOI).getListLength(data)); } else if (returnOI.getTypeName().equals(serdeConstants.VOID_TYPE_NAME)) { // null result.set(-1); } return result; }
case MAP: MapObjectInspector moi = (MapObjectInspector) oi; int mapSize = moi.getMapSize(val);
MapObjectInspector mapOI = (MapObjectInspector) inspector.getAllStructFieldRefs().get(10).getFieldObjectInspector(); assertEquals(3, mapOI.getMapSize(map)); assertEquals(4, mapOI.getMapValueElement(map, 2));
@Override public int size() { return _mapObjectInspector.getMapSize(_object); }
@Override public int size() { return _mapObjectInspector.getMapSize(_object); }
@Test public void testDeSerializeEmptyMap() throws Exception { // Test that timestamp object can be deserialized Writable[] wa = new Writable[] { new Text("{\"country\":\"Switzerland\",\"languages\":\"Italian\",\"religions\":\"\"}") }; for (Writable w : wa ) { JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); StructField sfr = soi.getStructFieldRef("religions"); assertEquals(sfr.getFieldObjectInspector().getCategory(), ObjectInspector.Category.MAP); MapObjectInspector moi = (MapObjectInspector) sfr.getFieldObjectInspector(); Object val = soi.getStructFieldData(result, sfr); assertEquals(-1, moi.getMapSize(val)); } }
@Test public void testDeSerializeMap() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"country\":\"Switzerland\",\"languages\":\"Italian\",\"religions\": { \"f\": \"v\", \"n\":null} }"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); StructField sfr = soi.getStructFieldRef("religions"); assertEquals(sfr.getFieldObjectInspector().getCategory(),ObjectInspector.Category.MAP); MapObjectInspector moi = (MapObjectInspector) sfr.getFieldObjectInspector(); Object val = soi.getStructFieldData(result, sfr) ; assertEquals(2, moi.getMapSize(val)); assertEquals("v", moi.getMapValueElement(val, "f")); ObjectInspector voi = moi.getMapValueObjectInspector(); assertTrue(voi instanceof StringObjectInspector); StringObjectInspector svoi = (StringObjectInspector) voi; assertNull( svoi.getPrimitiveJavaObject(moi.getMapValueElement(val,"n"))); }
@Override public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2) { int mapsize1 = moi1.getMapSize(o1); int mapsize2 = moi2.getMapSize(o2); if (mapsize1 != mapsize2) { return mapsize1 - mapsize2; } ObjectInspector mvoi1 = moi1.getMapValueObjectInspector(); ObjectInspector mvoi2 = moi2.getMapValueObjectInspector(); Map<?, ?> map1 = moi1.getMap(o1); for (Object mk1: map1.keySet()) { int rc = ObjectInspectorUtils.compare(moi1.getMapValueElement(o1, mk1), mvoi1, moi2.getMapValueElement(o2, mk1), mvoi2, this); if (rc != 0) { return rc; } } return 0; } }
private Object serializeMap(MapTypeInfo typeInfo, MapObjectInspector fieldOI, Object structFieldData, Schema schema) throws HaivvreoException { // Avro only allows maps with string keys if(!mapHasStringKey(fieldOI.getMapKeyObjectInspector())) throw new HaivvreoException("Avro only supports maps with keys as Strings. Current Map is: " + typeInfo.toString()); ObjectInspector mapKeyObjectInspector = fieldOI.getMapKeyObjectInspector(); ObjectInspector mapValueObjectInspector = fieldOI.getMapValueObjectInspector(); TypeInfo mapKeyTypeInfo = typeInfo.getMapKeyTypeInfo(); TypeInfo mapValueTypeInfo = typeInfo.getMapValueTypeInfo(); Map<?,?> map = fieldOI.getMap(structFieldData); Schema valueType = schema.getValueType(); Map<Object, Object> deserialized = new Hashtable<Object, Object>(fieldOI.getMapSize(structFieldData)); for (Map.Entry<?, ?> entry : map.entrySet()) { deserialized.put(serialize(mapKeyTypeInfo, mapKeyObjectInspector, entry.getKey(), null), // This works, but is a bit fragile. Construct a single String schema? serialize(mapValueTypeInfo, mapValueObjectInspector, entry.getValue(), valueType)); } return deserialized; }
private Object serializeMap(MapTypeInfo typeInfo, MapObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException { // Avro only allows maps with string keys if(!mapHasStringKey(fieldOI.getMapKeyObjectInspector())) { throw new AvroSerdeException("Avro only supports maps with keys as Strings. Current Map is: " + typeInfo.toString()); } ObjectInspector mapKeyObjectInspector = fieldOI.getMapKeyObjectInspector(); ObjectInspector mapValueObjectInspector = fieldOI.getMapValueObjectInspector(); TypeInfo mapKeyTypeInfo = typeInfo.getMapKeyTypeInfo(); TypeInfo mapValueTypeInfo = typeInfo.getMapValueTypeInfo(); Map<?,?> map = fieldOI.getMap(structFieldData); Schema valueType = schema.getValueType(); Map<Object, Object> deserialized = new HashMap<Object, Object>(fieldOI.getMapSize(structFieldData)); for (Map.Entry<?, ?> entry : map.entrySet()) { deserialized.put(serialize(mapKeyTypeInfo, mapKeyObjectInspector, entry.getKey(), STRING_SCHEMA), serialize(mapValueTypeInfo, mapValueObjectInspector, entry.getValue(), valueType)); } return deserialized; }
private Object serializeMap(MapTypeInfo typeInfo, MapObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException { // Avro only allows maps with string keys if(!mapHasStringKey(fieldOI.getMapKeyObjectInspector())) throw new AvroSerdeException("Avro only supports maps with keys as Strings. Current Map is: " + typeInfo.toString()); ObjectInspector mapKeyObjectInspector = fieldOI.getMapKeyObjectInspector(); ObjectInspector mapValueObjectInspector = fieldOI.getMapValueObjectInspector(); TypeInfo mapKeyTypeInfo = typeInfo.getMapKeyTypeInfo(); TypeInfo mapValueTypeInfo = typeInfo.getMapValueTypeInfo(); Map<?,?> map = fieldOI.getMap(structFieldData); Schema valueType = schema.getValueType(); Map<Object, Object> deserialized = new Hashtable<Object, Object>(fieldOI.getMapSize(structFieldData)); for (Map.Entry<?, ?> entry : map.entrySet()) { deserialized.put(serialize(mapKeyTypeInfo, mapKeyObjectInspector, entry.getKey(), null), // This works, but is a bit fragile. Construct a single String schema? serialize(mapValueTypeInfo, mapValueObjectInspector, entry.getValue(), valueType)); } return deserialized; }
private Object serializeMap(MapTypeInfo typeInfo, MapObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException { // Avro only allows maps with string keys if(!mapHasStringKey(fieldOI.getMapKeyObjectInspector())) { throw new AvroSerdeException("Avro only supports maps with keys as Strings. Current Map is: " + typeInfo.toString()); } ObjectInspector mapKeyObjectInspector = fieldOI.getMapKeyObjectInspector(); ObjectInspector mapValueObjectInspector = fieldOI.getMapValueObjectInspector(); TypeInfo mapKeyTypeInfo = typeInfo.getMapKeyTypeInfo(); TypeInfo mapValueTypeInfo = typeInfo.getMapValueTypeInfo(); Map<?,?> map = fieldOI.getMap(structFieldData); Schema valueType = schema.getValueType(); Map<Object, Object> deserialized = new LinkedHashMap<Object, Object>(fieldOI.getMapSize(structFieldData)); for (Map.Entry<?, ?> entry : map.entrySet()) { deserialized.put(serialize(mapKeyTypeInfo, mapKeyObjectInspector, entry.getKey(), STRING_SCHEMA), serialize(mapValueTypeInfo, mapValueObjectInspector, entry.getValue(), valueType)); } return deserialized; }
@Override public Object evaluate(DeferredObject[] arguments) throws HiveException { Object data = arguments[0].get(); if (returnOI.getCategory() == Category.MAP) { result.set(((MapObjectInspector) returnOI).getMapSize(data)); } else if (returnOI.getCategory() == Category.LIST) { result.set(((ListObjectInspector) returnOI).getListLength(data)); } else if (returnOI.getTypeName().equals(serdeConstants.VOID_TYPE_NAME)) { // null result.set(-1); } return result; }
@Override public Object evaluate(DeferredObject[] arguments) throws HiveException { Object data = arguments[0].get(); if (returnOI.getCategory() == Category.MAP) { result.set(((MapObjectInspector) returnOI).getMapSize(data)); } else if (returnOI.getCategory() == Category.LIST) { result.set(((ListObjectInspector) returnOI).getListLength(data)); } else if (returnOI.getTypeName().equals(Constants.VOID_TYPE_NAME)) { // null result.set(-1); } return result; }