soi = (StructObjectInspector) parameters[0]; countTruesField = soi.getStructFieldRef("counttrues"); countTruesFieldOI = (WritableLongObjectInspector) countTruesField.getFieldObjectInspector(); countFalsesField = soi.getStructFieldRef("countfalses"); countFalsesFieldOI = (WritableLongObjectInspector) countFalsesField.getFieldObjectInspector(); countNullsField = soi.getStructFieldRef("countnulls"); countNullsFieldOI = (WritableLongObjectInspector) countNullsField.getFieldObjectInspector(); partialResult[0] = new Text(); partialResult[1] = new LongWritable(0); partialResult[2] = new LongWritable(0); result[0] = new Text(); result[1] = new LongWritable(0); result[2] = new LongWritable(0); result[3] = new LongWritable(0); return ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi);
public static String getStandardStructTypeName(StructObjectInspector soi) { StringBuilder sb = new StringBuilder(); sb.append("struct<"); List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { if (i > 0) { sb.append(","); } sb.append(fields.get(i).getFieldName()); sb.append(":"); sb.append(fields.get(i).getFieldObjectInspector().getTypeName()); } sb.append(">"); return sb.toString(); }
public MyField(int fieldID, StructField field) { this.fieldID = fieldID; this.fieldName = field.getFieldName().toLowerCase(); this.fieldObjectInspector = field.getFieldObjectInspector(); this.fieldComment = field.getFieldComment(); }
private static void addInputColumnsToList(ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) { StructObjectInspector OI = shape.getOI(); for (StructField f : OI.getAllStructFieldRefs()) { fieldNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } }
public static StandardStructObjectInspector convertToStandardStructObjectInspector( StructObjectInspector structObjectInspector) throws HiveException { List<? extends StructField> fields = structObjectInspector.getAllStructFieldRefs(); List<ObjectInspector> oids = new ArrayList<ObjectInspector>(); ArrayList<String> columnNames = new ArrayList<String>(); for(StructField field : fields) { TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString( field.getFieldObjectInspector().getTypeName()); ObjectInspector standardWritableObjectInspector = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo); oids.add(standardWritableObjectInspector); columnNames.add(field.getFieldName()); } return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames,oids); }
private static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi) throws IOException { switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; sb.append("null"); } else { switch (poi.getPrimitiveCategory()) { case BOOLEAN: { boolean b = ((BooleanObjectInspector) poi).get(o); throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory()); List<? extends StructField> structFields = soi.getAllStructFieldRefs(); if (o == null) { sb.append("null"); sb.append(SerDeUtils.COMMA); appendWithQuotes(sb, structFields.get(i).getFieldName()); sb.append(SerDeUtils.COLON); buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)), structFields.get(i).getFieldObjectInspector());
return Murmur3.hash32(text.getBytes(), text.getLength()); return Murmur3.hash32(text.getBytes(), text.getLength()); + poi.getPrimitiveCategory()); int r = 0; ListObjectInspector listOI = (ListObjectInspector)objIns; ObjectInspector elemOI = listOI.getListElementObjectInspector(); for (int ii = 0; ii < listOI.getListLength(o); ++ii) { r = 31 * r + hashCodeMurmur(listOI.getListElement(o, ii), elemOI, byteBuffer); int r = 0; StructObjectInspector structOI = (StructObjectInspector)objIns; List<? extends StructField> fields = structOI.getAllStructFieldRefs(); for (StructField field : fields) { r = 31 * r + hashCodeMurmur(structOI.getStructFieldData(o, field), field.getFieldObjectInspector(), byteBuffer); byte tag = uOI.getTag(o); return hashCodeMurmur(uOI.getField(o), uOI.getObjectInspectors().get(tag), byteBuffer); throw new RuntimeException("Unknown type: "+ objIns.getTypeName());
switch (oi.getCategory()) { case PRIMITIVE: { return oi.getClass().getSimpleName(); ListObjectInspector loi = (ListObjectInspector) oi; return oi.getClass().getSimpleName() + "<" + getObjectInspectorName(loi.getListElementObjectInspector()) + ">"; + getObjectInspectorName(moi.getMapKeyObjectInspector()) + "," + getObjectInspectorName(moi.getMapValueObjectInspector()) + ">"; result.append(oi.getClass().getSimpleName() + "<"); StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { result.append(fields.get(i).getFieldName()); result.append(":"); result.append(getObjectInspectorName(fields.get(i) .getFieldObjectInspector())); if (i == fields.size() - 1) { result.append(">"); result.append(oi.getClass().getSimpleName() + "<"); UnionObjectInspector uoi = (UnionObjectInspector)oi; List<ObjectInspector> ois = uoi.getObjectInspectors(); for (int i = 0; i < ois.size(); i++) { if (i > 0) {
List<? extends StructField> structFields = soi.getAllStructFieldRefs(); assert (columnNames.size() == structFields.size()); if (obj == null) { buildJSONString(sb, soi.getStructFieldData(obj, structFields.get(i)), structFields.get(i).getFieldObjectInspector()); throw new SerDeException(e); return new Text(sb.toString());
throws IOException, SerDeException { if (obj == null) { out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); return; switch (objInspector.getCategory()) { case PRIMITIVE: PrimitiveObjectInspector oi = (PrimitiveObjectInspector) objInspector; if (oi.getPrimitiveCategory() == PrimitiveCategory.BINARY) { BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(obj); byte[] toWrite = new byte[bw.getLength()]; ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map<?, ?> map = moi.getMap(obj); separator = (char) separators[level]; StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); list = soi.getStructFieldsDataAsList(obj); out.write(separator); serializeNoEncode(out, list.get(i), fields.get(i).getFieldObjectInspector(), separators, level + 1, nullSequence, escaped, escapeChar, needsEscape); throw new RuntimeException("Unknown category type: "+ objInspector.getCategory());
out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); return; switch (objInspector.getCategory()) { case PRIMITIVE: LazyUtils.writePrimitiveUTF8(out, obj, ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map<?, ?> map = moi.getMap(obj); separator = (char) LazyUtils.getSeparator(separators, level); StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); list = soi.getStructFieldsDataAsList(obj); out.write(separator); serialize(out, list.get(i), fields.get(i).getFieldObjectInspector(), separators, level + 1, nullSequence, escaped, escapeChar, needsEscape); separator = (char) LazyUtils.getSeparator(separators, level); UnionObjectInspector uoi = (UnionObjectInspector) objInspector; List<? extends ObjectInspector> ois = uoi.getObjectInspectors(); + objInspector.getCategory());
switch (outputOI.getCategory()) { case PRIMITIVE: primOutputOI.getTypeInfo()); case STRUCT: StructObjectInspector structOutputOI = (StructObjectInspector) outputOI; List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs(); List<String> structFieldNames = new ArrayList<String>(listFields.size()); List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>( structFieldNames.add(listField.getFieldName()); structFieldObjectInspectors.add(getConvertedOI(listField.getFieldObjectInspector(), listField.getFieldObjectInspector(), oiSettableProperties, false)); return ObjectInspectorFactory.getStandardStructObjectInspector( structFieldNames, structFieldObjectInspectors); ListObjectInspector listOutputOI = (ListObjectInspector) outputOI; UnionObjectInspector unionOutputOI = (UnionObjectInspector) outputOI; + inputOI.getTypeName() + " to " + outputOI.getTypeName() + " not supported yet.");
switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector loi = (PrimitiveObjectInspector) oi; if (objectInspectorOption == ObjectInspectorCopyOption.DEFAULT) { objectInspectorOption = loi.preferWritable() ? ObjectInspectorCopyOption.WRITABLE : ObjectInspectorCopyOption.JAVA; result = loi.getPrimitiveJavaObject(o); if (loi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP) { int length = loi.getListLength(o); ArrayList<Object> list = new ArrayList<Object>(length); for (int i = 0; i < length; i++) { list.add(copyToStandardObject(loi.getListElement(o, i), loi .getListElementObjectInspector(), objectInspectorOption)); List<? extends StructField> fields = soi.getAllStructFieldRefs(); ArrayList<Object> struct = new ArrayList<Object>(fields.size()); for (StructField f : fields) { struct.add(copyToStandardObject(soi.getStructFieldData(o, f), f .getFieldObjectInspector(), objectInspectorOption)); List<ObjectInspector> objectInspectors = uoi.getObjectInspectors(); Object object = copyToStandardObject( uoi.getField(o), objectInspectors.get(uoi.getTag(o)), objectInspectorOption); result = object;
@Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { if (objInspector.getCategory() != Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); } StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < fields.size(); i++) { if (i > 0) { sb.append(separator); } Object column = soi.getStructFieldData(obj, fields.get(i)); if (fields.get(i).getFieldObjectInspector().getCategory() == Category.PRIMITIVE) { // For primitive object, serialize to plain string sb.append(column == null ? nullString : column.toString()); } else { // For complex object, serialize to JSON format sb.append(SerDeUtils.getJSONString(column, fields.get(i) .getFieldObjectInspector())); } } serializeCache.set(sb.toString()); return serializeCache; }
switch (oi.getCategory()) { case PRIMITIVE: { if (o == null) { ListObjectInspector loi = (ListObjectInspector) oi; ObjectInspector listElementObjectInspector = loi .getListElementObjectInspector(); List<?> olist = loi.getList(o); if (olist == null) { return true; List<? extends StructField> structFields = soi.getAllStructFieldRefs(); if (o == null) { return true; if (hasAnyNullObject(soi.getStructFieldData(o, structFields.get(i)), structFields.get(i).getFieldObjectInspector())) { return true; } else { if (uoi.getObjectInspectors().size() == 0) { return false; return hasAnyNullObject(uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)));
private ObjectInspector convertStruct(ObjectInspector inspector, AtomicBoolean foundUnion) { StructObjectInspector structOI = (StructObjectInspector) inspector; List<? extends StructField> fields = structOI.getAllStructFieldRefs(); List<String> names = new ArrayList<>(fields.size()); List<ObjectInspector> inspectors = new ArrayList<>(fields.size()); for (StructField field : fields) { names.add(field.getFieldName()); inspectors.add(convert(field.getFieldObjectInspector(), foundUnion)); } return ObjectInspectorFactory.getStandardStructObjectInspector(names, inspectors); }
ObjectInspector.Category oiCategory = oi.getCategory(); switch (oiCategory) { case PRIMITIVE: case LIST: ListObjectInspector loi = (ListObjectInspector) oi; int listSize = loi.getListLength(val); ObjectInspector listElementOI = loi.getListElementObjectInspector(); for (int idx = 0; idx < listSize; ++idx) { convertedList.add(convertValue(loi.getListElement(val, idx), listElementOI)); case MAP: MapObjectInspector moi = (MapObjectInspector) oi; int mapSize = moi.getMapSize(val); ObjectInspector mapKeyOI = moi.getMapKeyObjectInspector(); ObjectInspector mapValOI = moi.getMapValueObjectInspector(); Map<?, ?> mapCol = moi.getMap(val); for (Object mapKey : mapCol.keySet()) { StructObjectInspector soi = (StructObjectInspector) oi; List<Object> convertedRow = new ArrayList<Object>(); for (StructField structField : soi.getAllStructFieldRefs()) { Object convertedFieldValue = convertValue( soi.getStructFieldData(val, structField), structField.getFieldObjectInspector()); convertedRow.add(convertedFieldValue);
StructField sf = soi.getStructFieldRef(names[i]); if (sf == null) { throw new MetaException("Invalid Field " + names[i]); } else { oi = sf.getFieldObjectInspector(); if (oi.getCategory() != Category.STRUCT) { str_fields.add(new FieldSchema(last_name, oi.getTypeName(), FROM_SERIALIZER)); } else { List<? extends StructField> fields = ((StructObjectInspector) oi) .getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { StructField structField = fields.get(i); String fieldName = structField.getFieldName(); String fieldTypeName = structField.getFieldObjectInspector().getTypeName(); String fieldComment = determineFieldComment(structField.getFieldComment());
throws SerDeException { if (objInspector.getCategory() != Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); List<? extends StructField> fields = soi.getAllStructFieldRefs(); List<Object> list = soi.getStructFieldsDataAsList(obj); List<? extends StructField> declaredFields = (serdeParams.getRowTypeInfo() != null && ((StructTypeInfo) serdeParams.getRowTypeInfo()) .getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector()) .getAllStructFieldRefs() : null; ObjectInspector foi = fields.get(i).getFieldObjectInspector(); Object f = (list == null ? null : list.get(i)); + "tableType=" + serdeParams.getRowTypeInfo().toString() + "\n" + "dataType=" + TypeInfoUtils.getTypeInfoFromObjectInspector(objInspector)); .set(serializeStream.getData(), 0, serializeStream.getLength()); serializedSize = serializeStream.getLength(); lastOperationSerialize = true;
switch (oi.getCategory()) { case PRIMITIVE: Writable writable = (Writable) ((PrimitiveObjectInspector) oi).getPrimitiveWritableObject(data); return writableWriter.write(writable, generator); generator.writeBeginArray(); for (Map.Entry<?, ?> entry : moi.getMap(data).entrySet()) { Result result = write(entry.getValue(), moi.getMapValueObjectInspector(), generator); if (!result.isSuccesful()) { return result; StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> refs = soi.getAllStructFieldRefs(); String actualFieldName = alias.toES(structField.getFieldName()); if (shouldKeep(generator.getParentPath(), actualFieldName)) { generator.writeFieldName(actualFieldName); Result result = write(soi.getStructFieldData(data, structField), structField.getFieldObjectInspector(), generator); if (!result.isSuccesful()) { return result;