switch (oi.getCategory()) { case PRIMITIVE: { return oi.getClass().getSimpleName(); ListObjectInspector loi = (ListObjectInspector) oi; return oi.getClass().getSimpleName() + "<" + getObjectInspectorName(loi.getListElementObjectInspector()) + ">"; + getObjectInspectorName(moi.getMapKeyObjectInspector()) + "," + getObjectInspectorName(moi.getMapValueObjectInspector()) + ">"; result.append(oi.getClass().getSimpleName() + "<"); StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { result.append(fields.get(i).getFieldName()); result.append(":"); result.append(getObjectInspectorName(fields.get(i) .getFieldObjectInspector())); if (i == fields.size() - 1) { result.append(">"); result.append(oi.getClass().getSimpleName() + "<"); UnionObjectInspector uoi = (UnionObjectInspector)oi; List<ObjectInspector> ois = uoi.getObjectInspectors(); for (int i = 0; i < ois.size(); i++) { if (i > 0) {
switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector loi = (PrimitiveObjectInspector) oi; int length = loi.getListLength(o); ArrayList<Object> list = new ArrayList<Object>(length); for (int i = 0; i < length; i++) { list.add(copyToStandardObject(loi.getListElement(o, i), loi .getListElementObjectInspector(), objectInspectorOption)); List<? extends StructField> fields = soi.getAllStructFieldRefs(); ArrayList<Object> struct = new ArrayList<Object>(fields.size()); for (StructField f : fields) { struct.add(copyToStandardObject(soi.getStructFieldData(o, f), f .getFieldObjectInspector(), objectInspectorOption)); List<ObjectInspector> objectInspectors = uoi.getObjectInspectors(); Object object = copyToStandardObject( uoi.getField(o), objectInspectors.get(uoi.getTag(o)), objectInspectorOption); result = object;
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { if (arguments.length == 1) { sourceOI = arguments[0]; return objectInspectorConverter.convert(sourceOI); } if (arguments.length == 2 && (arguments[0] instanceof UnionObjectInspector) && (arguments[1] instanceof WritableConstantIntObjectInspector)) { tag = ((WritableConstantIntObjectInspector) arguments[1]).getWritableConstantValue().get(); unionOI = (UnionObjectInspector) arguments[0]; List<ObjectInspector> fieldOIs = ((UnionObjectInspector) arguments[0]).getObjectInspectors(); if (tag < 0 || tag >= fieldOIs.size()) { throw new UDFArgumentException( "int constant must be a valid union tag for " + unionOI.getTypeName() + ". Expected 0-" + (fieldOIs.size() - 1) + " got: " + tag); } return fieldOIs.get(tag); } String argumentTypes = "nothing"; if (arguments.length > 0) { List<String> typeNames = new ArrayList<>(); for (ObjectInspector oi : arguments) { typeNames.add(oi.getTypeName()); } argumentTypes = typeNames.toString(); } throw new UDFArgumentException( "Unsupported arguments. Expected a type containing a union or a union and an int constant, got: " + argumentTypes); }
private static void serializeUnion(RandomAccessOutput byteStream, Object obj, UnionObjectInspector uoi, BooleanRef warnedOnceNullMapKey) throws SerDeException { byte tag = uoi.getTag(obj); byteStream.write(tag); serialize(byteStream, uoi.getField(obj), uoi.getObjectInspectors().get(tag), false, warnedOnceNullMapKey); }
switch (oi.getCategory()) { case PRIMITIVE: { if (o == null) { ListObjectInspector loi = (ListObjectInspector) oi; ObjectInspector listElementObjectInspector = loi .getListElementObjectInspector(); List<?> olist = loi.getList(o); if (olist == null) { return true; List<? extends StructField> structFields = soi.getAllStructFieldRefs(); if (o == null) { return true; if (hasAnyNullObject(soi.getStructFieldData(o, structFields.get(i)), structFields.get(i).getFieldObjectInspector())) { return true; } else { if (uoi.getObjectInspectors().size() == 0) { return false; return hasAnyNullObject(uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)));
switch (oi.getCategory()) { case PRIMITIVE: return true; case LIST: ListObjectInspector loi = (ListObjectInspector) oi; return compareSupported(loi.getListElementObjectInspector()); case STRUCT: StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int f = 0; f < fields.size(); f++) { if (!compareSupported(fields.get(f).getFieldObjectInspector())) { return false; case UNION: UnionObjectInspector uoi = (UnionObjectInspector) oi; for (ObjectInspector eoi : uoi.getObjectInspectors()) { if (!compareSupported(eoi)) { return false;
if (oi1.getCategory() != oi2.getCategory()) { return oi1.getCategory().compareTo(oi2.getCategory()); ListObjectInspector loi1 = (ListObjectInspector) oi1; ListObjectInspector loi2 = (ListObjectInspector) oi2; int minimum = Math.min(loi1.getListLength(o1), loi2.getListLength(o2)); for (int i = 0; i < minimum; i++) { int r = compare(loi1.getListElement(o1, i), loi1 .getListElementObjectInspector(), loi2.getListElement(o2, i), loi2 .getListElementObjectInspector(), UnionObjectInspector uoi1 = (UnionObjectInspector) oi1; UnionObjectInspector uoi2 = (UnionObjectInspector) oi2; byte tag1 = uoi1.getTag(o1); byte tag2 = uoi2.getTag(o2); if (tag1 != tag2) { return tag1 - tag2; return compare(uoi1.getField(o1), uoi1.getObjectInspectors().get(tag1), uoi2.getField(o2), uoi2.getObjectInspectors().get(tag2), mapEqualComparer, nullValueOpt);
int mapEntries = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_STATS_MAP_NUM_ENTRIES); switch (oi.getCategory()) { case PRIMITIVE: String colTypeLowerCase = oi.getTypeName().toLowerCase(); if (colTypeLowerCase.equals(serdeConstants.STRING_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.VARCHAR_TYPE_NAME) if (leoi.getCategory().equals(ObjectInspector.Category.PRIMITIVE)) { result += getSizeOfPrimitiveTypeArraysFromType(leoi.getTypeName(), length, conf); } else { result += soi.getAllStructFieldRefs().size() * JavaDataModel.get().ref(); for (StructField field : soi.getAllStructFieldRefs()) { result += getSizeOfComplexTypes(conf, field.getFieldObjectInspector()); result += uoi.getObjectInspectors().size() * JavaDataModel.get().primitive1(); for (ObjectInspector foi : uoi.getObjectInspectors()) { result += getSizeOfComplexTypes(conf, foi);
switch (objectInspector.getCategory()) { case PRIMITIVE: return toAvroSchema((PrimitiveObjectInspector) objectInspector); ListObjectInspector listObjectInspector = (ListObjectInspector) objectInspector; return Schema.createArray( toAvroSchema(listObjectInspector.getListElementObjectInspector())); case MAP: MapObjectInspector mapObjectInspector = (MapObjectInspector) objectInspector; return Schema.createMap(toAvroSchema(mapObjectInspector.getMapValueObjectInspector())); case STRUCT: List structFields = ((StructObjectInspector) objectInspector).getAllStructFieldRefs(); List<Schema.Field> fields = Lists.newArrayList(); for (Object structFieldObj : structFields) { StructField structField = (StructField) structFieldObj; String fieldName = structField.getFieldName(); String fieldComment = structField.getFieldComment(); Schema.Field field = new Schema.Field(fieldName, toAvroSchema(structField.getFieldObjectInspector()), fieldComment, null); fields.add(field); case UNION: UnionObjectInspector unionObjectInspector = (UnionObjectInspector) objectInspector; List<ObjectInspector> unionObjectInspectors = unionObjectInspector.getObjectInspectors(); List<Schema> unionTypes = Lists.newArrayList(); for (ObjectInspector unionSubObjectInspector : unionObjectInspectors) {
listColumnVector.child, listTypeInfo.getListElementTypeInfo(), listObjectInspector.getListElementObjectInspector(), offset + i)); mapColumnVector.keys, mapTypeInfo.getMapKeyTypeInfo(), mapObjectInspector.getMapKeyObjectInspector(), offset + i); final Object value = extractRowColumn( mapColumnVector.values, mapTypeInfo.getMapValueTypeInfo(), mapObjectInspector.getMapValueObjectInspector(), offset + i); map.put(key, value); structColumnVector.fields[i], fieldTypeInfo, structField.getFieldObjectInspector(), adjustedIndex); structInspector.setStructFieldData(struct, structField, value); final List<TypeInfo> objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos(); final UnionObjectInspector unionInspector = (UnionObjectInspector) objectInspector; final List<ObjectInspector> unionInspectors = unionInspector.getObjectInspectors(); final UnionColumnVector unionColumnVector = (UnionColumnVector) colVector; final byte tag = (byte) unionColumnVector.tags[adjustedIndex];
StructField sfr = soi.getStructFieldRef("stuff"); assertEquals(sfr.getFieldObjectInspector().getCategory(), ObjectInspector.Category.UNION); UnionObjectInspector uoi = (UnionObjectInspector) sfr.getFieldObjectInspector(); Object val = soi.getStructFieldData(result, sfr) ; assertEquals("Italian", uoi.getField(val)); uoi.getTypeName(); val = soi.getStructFieldData(result, sfr) ; assertEquals("2", val); assertEquals(0, uoi.getTag(val)); val = soi.getStructFieldData(result, sfr) ; assertTrue(val instanceof JSONObject); assertEquals(3, uoi.getTag(val)); val = soi.getStructFieldData(result, sfr) ; assertTrue(val instanceof JSONArray); assertEquals(2, uoi.getTag(val));
public static String getStandardUnionTypeName(UnionObjectInspector uoi) { StringBuilder sb = new StringBuilder(); sb.append(serdeConstants.UNION_TYPE_NAME + "<"); List<ObjectInspector> ois = uoi.getObjectInspectors(); for(int i = 0; i < ois.size(); i++) { if (i > 0) { sb.append(","); } sb.append(ois.get(i).getTypeName()); } sb.append(">"); return sb.toString(); }
simpleCase = true; inspector = (StructObjectInspector) rowInspector; field = inspector.getStructFieldRef(names[0]); return outputOI = field.getFieldObjectInspector(); if (unionIndex == -1) { inspectors[i] = (StructObjectInspector) fields[i - 1] .getFieldObjectInspector(); } else { inspectors[i] = (StructObjectInspector) ( (UnionObjectInspector)fields[i-1].getFieldObjectInspector()). getObjectInspectors().get(unionIndex); fields[i] = inspectors[i].getStructFieldRef(unionfields[0]); if (unionfields.length > 1) { unionIndex = Integer.parseInt(unionfields[1]);
@Test public void evaluate_UnionAndTagArguments_NotMatchesTag() throws HiveException { when(unionOI.getObjectInspectors()).thenReturn( ImmutableList.<ObjectInspector> of( PrimitiveObjectInspectorFactory.writableStringObjectInspector, PrimitiveObjectInspectorFactory.writableLongObjectInspector)); when(tagOI.getWritableConstantValue()).thenReturn(new IntWritable(0)); when(deferredObject.get()).thenReturn(value); when(unionOI.getTag(value)).thenReturn((byte) 1); underTest.initialize(new ObjectInspector[] { unionOI, tagOI }); Object result = underTest.evaluate(new DeferredObject[] { deferredObject }); assertThat(result, is(nullValue())); }
private ObjectInspector convertUnion(ObjectInspector inspector, AtomicBoolean foundUnion) { UnionObjectInspector unionOI = (UnionObjectInspector) inspector; List<ObjectInspector> fieldOIs = unionOI.getObjectInspectors(); int tags = fieldOIs.size(); List<String> names = new ArrayList<>(tags); List<ObjectInspector> inspectors = new ArrayList<>(tags); for (int i = 0; i < tags; i++) { names.add(TAG_FIELD_PREFIX + i); inspectors.add(convert(fieldOIs.get(i), foundUnion)); } return ObjectInspectorFactory.getStandardStructObjectInspector(names, inspectors); }
static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, String nullStr) { switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; ListObjectInspector loi = (ListObjectInspector) oi; ObjectInspector listElementObjectInspector = loi .getListElementObjectInspector(); List<?> olist = loi.getList(o); if (olist == null) { sb.append(nullStr); List<? extends StructField> structFields = soi.getAllStructFieldRefs(); if (o == null) { sb.append(nullStr); sb.append(structFields.get(i).getFieldName()); sb.append(QUOTE); sb.append(COLON); buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)), structFields.get(i).getFieldObjectInspector(), JSON_NULL); } else { sb.append(LBRACE); sb.append(uoi.getTag(o)); sb.append(COLON); buildJSONString(sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)), JSON_NULL); sb.append(RBRACE);
switch (oi.getCategory()) { case PRIMITIVE: return true; case LIST: ListObjectInspector loi = (ListObjectInspector) oi; return compareSupported(loi.getListElementObjectInspector()); case STRUCT: StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int f = 0; f < fields.size(); f++) { if (!compareSupported(fields.get(f).getFieldObjectInspector())) { return false; case UNION: UnionObjectInspector uoi = (UnionObjectInspector) oi; for (ObjectInspector eoi : uoi.getObjectInspectors()) { if (!compareSupported(eoi)) { return false;
int mapEntries = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_STATS_MAP_NUM_ENTRIES); switch (oi.getCategory()) { case PRIMITIVE: String colTypeLowerCase = oi.getTypeName().toLowerCase(); if (colTypeLowerCase.equals(serdeConstants.STRING_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.VARCHAR_TYPE_NAME) if (leoi.getCategory().equals(ObjectInspector.Category.PRIMITIVE)) { result += getSizeOfPrimitiveTypeArraysFromType(leoi.getTypeName(), length); } else { result += soi.getAllStructFieldRefs().size() * JavaDataModel.get().ref(); for (StructField field : soi.getAllStructFieldRefs()) { result += getSizeOfComplexTypes(conf, field.getFieldObjectInspector()); result += uoi.getObjectInspectors().size() * JavaDataModel.get().primitive1(); for (ObjectInspector foi : uoi.getObjectInspectors()) { result += getSizeOfComplexTypes(conf, foi);
public UnionConverter(ObjectInspector inputOI, SettableUnionObjectInspector outputOI) { if (inputOI instanceof UnionObjectInspector) { this.inputOI = (UnionObjectInspector)inputOI; this.outputOI = outputOI; inputTagsOIs = this.inputOI.getObjectInspectors(); outputTagsOIs = outputOI.getObjectInspectors(); // If the output has some extra fields, set them to NULL in convert(). int minFields = Math.min(inputTagsOIs.size(), outputTagsOIs.size()); fieldConverters = new ArrayList<Converter>(minFields); for (int f = 0; f < minFields; f++) { fieldConverters.add(getConverter(inputTagsOIs.get(f), outputTagsOIs.get(f))); } // Create an empty output object which will be populated when convert() is invoked. output = outputOI.create(); } else if (!(inputOI instanceof VoidObjectInspector)) { throw new RuntimeException("Hive internal error: conversion of " + inputOI.getTypeName() + " to " + outputOI.getTypeName() + "not supported yet."); } }