Refine search
/** * Helper method to return underlying Java Map from * an object-representation that is readable by a provided * MapObjectInspector */ private static Map<?, ?> serializeMap(Object f, MapObjectInspector moi) throws SerDeException { ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map<Object, Object> m = new LinkedHashMap<Object, Object>(); Map<?, ?> readMap = moi.getMap(f); if (readMap == null) { return null; } else { for (Map.Entry<?, ?> entry : readMap.entrySet()) { m.put(serializeField(entry.getKey(), koi), serializeField(entry.getValue(), voi)); } } return m; }
private ObjectInspector convertMap(ObjectInspector inspector, AtomicBoolean foundUnion) { MapObjectInspector mapOI = (MapObjectInspector) inspector; ObjectInspector keyOI = convert(mapOI.getMapKeyObjectInspector(), foundUnion); ObjectInspector valueOI = convert(mapOI.getMapValueObjectInspector(), foundUnion); return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI); }
private Object parseMap(JsonParser parser, MapObjectInspector oi) throws IOException, SerDeException { if (parser.getCurrentToken() == JsonToken.VALUE_NULL) { parser.nextToken(); return null; } Map<Object, Object> ret = new LinkedHashMap<>(); if (parser.getCurrentToken() != JsonToken.START_OBJECT) { throw new SerDeException("struct expected"); } if (!(oi.getMapKeyObjectInspector() instanceof PrimitiveObjectInspector)) { throw new SerDeException("map key must be a primitive"); } PrimitiveObjectInspector keyOI = (PrimitiveObjectInspector) oi.getMapKeyObjectInspector(); ObjectInspector valOI = oi.getMapValueObjectInspector(); JsonToken currentToken = parser.nextToken(); while (currentToken != null && currentToken != JsonToken.END_OBJECT) { if (currentToken != JsonToken.FIELD_NAME) { throw new SerDeException("unexpected token: " + currentToken); } Object key = parseMapKey(parser, keyOI); Object val = parseDispatcher(parser, valOI); ret.put(key, val); currentToken = parser.getCurrentToken(); } if (currentToken != null) { parser.nextToken(); } return ret; }
private ObjectInspector convertMap(ObjectInspector inspector, AtomicBoolean foundUnion) { MapObjectInspector mapOI = (MapObjectInspector) inspector; ObjectInspector keyOI = convert(mapOI.getMapKeyObjectInspector(), foundUnion); ObjectInspector valueOI = convert(mapOI.getMapValueObjectInspector(), foundUnion); return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI); }
@Override public StructObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException { if (args.length != 1) { throw new UDFArgumentException("explode() takes only one argument"); } ArrayList<String> fieldNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); switch (args[0].getCategory()) { case LIST: inputOI = args[0]; fieldNames.add("col"); fieldOIs.add(((ListObjectInspector)inputOI).getListElementObjectInspector()); break; case MAP: inputOI = args[0]; fieldNames.add("key"); fieldNames.add("value"); fieldOIs.add(((MapObjectInspector)inputOI).getMapKeyObjectInspector()); fieldOIs.add(((MapObjectInspector)inputOI).getMapValueObjectInspector()); break; default: throw new UDFArgumentException("explode() takes an array or a map as a parameter"); } return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
private void getMapObject(String parentName, MapObjectInspector mapObjectInspector, Object mapObject, Map<String, Object> parsedMap) { if (mapObjectInspector.getMapSize(mapObject) < 0) { return; } String mapChildFieldNameFormat = StringUtils.replace( StringUtils.format(mapParentFieldNameFormat, parentName), MAP_CHILD_TAG, "%s" ); Map objectMap = mapObjectInspector.getMap(mapObject); PrimitiveObjectInspector key = (PrimitiveObjectInspector) mapObjectInspector.getMapKeyObjectInspector(); PrimitiveObjectInspector value = (PrimitiveObjectInspector) mapObjectInspector.getMapValueObjectInspector(); objectMap.forEach((k, v) -> { String resolvedFieldName = StringUtils.format(mapChildFieldNameFormat, key.getPrimitiveJavaObject(k).toString()); parsedMap.put(resolvedFieldName, value.getPrimitiveJavaObject(v)); }); }
public MapConverter(ObjectInspector inputOI, SettableMapObjectInspector outputOI) { if (inputOI instanceof MapObjectInspector) { this.inputOI = (MapObjectInspector)inputOI; this.outputOI = outputOI; inputKeyOI = this.inputOI.getMapKeyObjectInspector(); outputKeyOI = outputOI.getMapKeyObjectInspector(); inputValueOI = this.inputOI.getMapValueObjectInspector(); outputValueOI = outputOI.getMapValueObjectInspector(); keyConverters = new ArrayList<Converter>(); valueConverters = new ArrayList<Converter>(); output = outputOI.create(); } else if (!(inputOI instanceof VoidObjectInspector)) { throw new RuntimeException("Hive internal error: conversion of " + inputOI.getTypeName() + " to " + outputOI.getTypeName() + "not supported yet."); } }
/** * Convert the given object to a lazy object using the given {@link ObjectInspector} * * @param obj Object to be converted to a {@link LazyObject} * @param objectInspector ObjectInspector used for the conversion * @return the created {@link LazyObject lazy object} * */ @SuppressWarnings({ "rawtypes", "unchecked" }) private Object toLazyMapObject(Object obj, ObjectInspector objectInspector) { if (obj == null) { return null; } // avro guarantees that the key will be of type string. So we just need to worry about // deserializing the value here LazyMap lazyMap = (LazyMap) LazyFactory.createLazyObject(objectInspector); Map map = lazyMap.getMap(); Map<Object, Object> origMap = (Map) obj; ObjectInspector keyObjectInspector = ((MapObjectInspector) objectInspector).getMapKeyObjectInspector(); ObjectInspector valueObjectInspector = ((MapObjectInspector) objectInspector).getMapValueObjectInspector(); for (Entry entry : origMap.entrySet()) { Object value = entry.getValue(); map.put(toLazyPrimitiveObject(entry.getKey(), keyObjectInspector), toLazyObject(value, valueObjectInspector)); } return lazyMap; }
Type keyType = typeParameters.get(0); Type valueType = typeParameters.get(1); ObjectInspector keyInspector = inspector.getMapKeyObjectInspector(); ObjectInspector valueInspector = inspector.getMapValueObjectInspector(); BlockBuilder currentBuilder;
@Override public StructObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException { if (args.length != 1) { throw new UDFArgumentException("explode() takes only one argument"); } ArrayList<String> fieldNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); switch (args[0].getCategory()) { case LIST: inputOI = args[0]; fieldNames.add("col"); fieldOIs.add(((ListObjectInspector)inputOI).getListElementObjectInspector()); break; case MAP: inputOI = args[0]; fieldNames.add("key"); fieldNames.add("value"); fieldOIs.add(((MapObjectInspector)inputOI).getMapKeyObjectInspector()); fieldOIs.add(((MapObjectInspector)inputOI).getMapValueObjectInspector()); break; default: throw new UDFArgumentException("explode() takes an array or a map as a parameter"); } return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
return mapsize1 - mapsize2; ObjectInspector mkoi1 = moi1.getMapKeyObjectInspector(); ObjectInspector mkoi2 = moi2.getMapKeyObjectInspector(); ObjectInspector mvoi1 = moi1.getMapValueObjectInspector(); ObjectInspector mvoi2 = moi2.getMapValueObjectInspector();
return mapsize1 - mapsize2; ObjectInspector mkoi1 = moi1.getMapKeyObjectInspector(); ObjectInspector mkoi2 = moi2.getMapKeyObjectInspector(); ObjectInspector mvoi1 = moi1.getMapValueObjectInspector(); ObjectInspector mvoi2 = moi2.getMapValueObjectInspector();
private Object serializeMap(MapTypeInfo typeInfo, MapObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException { // Avro only allows maps with string keys if(!mapHasStringKey(fieldOI.getMapKeyObjectInspector())) { throw new AvroSerdeException("Avro only supports maps with keys as Strings. Current Map is: " + typeInfo.toString()); } ObjectInspector mapKeyObjectInspector = fieldOI.getMapKeyObjectInspector(); ObjectInspector mapValueObjectInspector = fieldOI.getMapValueObjectInspector(); TypeInfo mapKeyTypeInfo = typeInfo.getMapKeyTypeInfo(); TypeInfo mapValueTypeInfo = typeInfo.getMapValueTypeInfo(); Map<?,?> map = fieldOI.getMap(structFieldData); Schema valueType = schema.getValueType(); Map<Object, Object> deserialized = new LinkedHashMap<Object, Object>(fieldOI.getMapSize(structFieldData)); for (Map.Entry<?, ?> entry : map.entrySet()) { deserialized.put(serialize(mapKeyTypeInfo, mapKeyObjectInspector, entry.getKey(), STRING_SCHEMA), serialize(mapValueTypeInfo, mapValueObjectInspector, entry.getValue(), valueType)); } return deserialized; }
MapObjectInspector moi = (MapObjectInspector) oi; return oi.getClass().getSimpleName() + "<" + getObjectInspectorName(moi.getMapKeyObjectInspector()) + "," + getObjectInspectorName(moi.getMapValueObjectInspector()) + ">";
MapTreeWriter(int columnId, ObjectInspector inspector, StreamFactory writer, boolean nullable) throws IOException { super(columnId, inspector, writer, nullable); this.isDirectV2 = isNewWriteFormat(writer); MapObjectInspector insp = (MapObjectInspector) inspector; childrenWriters = new TreeWriter[2]; childrenWriters[0] = createTreeWriter(insp.getMapKeyObjectInspector(), writer, true); childrenWriters[1] = createTreeWriter(insp.getMapValueObjectInspector(), writer, true); lengths = createIntegerWriter(writer.createStream(columnId, OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer); recordPosition(rowIndexPosition); }
keyIsNull[i] = false; LazyBinaryUtils.checkObjectByteInfo(((MapObjectInspector) oi) .getMapKeyObjectInspector(), bytes, lastElementByteEnd, recordInfo, vInt); keyStart[i] = lastElementByteEnd + recordInfo.elementOffset; keyLength[i] = recordInfo.elementSize; valueIsNull[i] = false; LazyBinaryUtils.checkObjectByteInfo(((MapObjectInspector) oi) .getMapValueObjectInspector(), bytes, lastElementByteEnd, recordInfo, vInt); valueStart[i] = lastElementByteEnd + recordInfo.elementOffset;
boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) { MapObjectInspector moi = (MapObjectInspector) objInspector; ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map<?, ?> map = moi.getMap(obj);
public MapDataWriter(MapObjectInspector inspector, GroupType groupType) { this.inspector = inspector; // Get the internal map structure (MAP_KEY_VALUE) GroupType repeatedType = groupType.getType(0).asGroupType(); this.repeatedGroupName = repeatedType.getName(); // Get key element information Type keyType = repeatedType.getType(0); ObjectInspector keyInspector = this.inspector.getMapKeyObjectInspector(); this.keyName = keyType.getName(); this.keyWriter = createWriter(keyInspector, keyType); // Get value element information Type valuetype = repeatedType.getType(1); ObjectInspector valueInspector = this.inspector.getMapValueObjectInspector(); this.valueName = valuetype.getName(); this.valueWriter = createWriter(valueInspector, valuetype); }
ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector(); ObjectInspector mapValueObjectInspector = moi .getMapValueObjectInspector(); Map<?, ?> omap = moi.getMap(o); if (omap == null) {
ObjectInspector mapKeyOI = moi.getMapKeyObjectInspector(); ObjectInspector mapValOI = moi.getMapValueObjectInspector(); Map<?, ?> mapCol = moi.getMap(val); for (Object mapKey : mapCol.keySet()) {