protected static ObjectInspector[] getObjectInspectorsForBucketedCols(List<Integer> bucketIds , StructObjectInspector recordObjInspector) { ObjectInspector[] result = new ObjectInspector[bucketIds.size()]; for (int i = 0; i < bucketIds.size(); i++) { int bucketId = bucketIds.get(i); result[i] = recordObjInspector.getAllStructFieldRefs().get(bucketId).getFieldObjectInspector(); } return result; }
private static void addInputColumnsToList(ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) { StructObjectInspector OI = shape.getOI(); for (StructField f : OI.getAllStructFieldRefs()) { fieldNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } }
public static boolean isSupportedKey(ObjectInspector keyOi) { List<? extends StructField> keyFields = ((StructObjectInspector)keyOi).getAllStructFieldRefs(); for (StructField field : keyFields) { if (!MapJoinKey.isSupportedField(field.getFieldObjectInspector())) { return false; } } return true; }
public static VectorExpressionWriter[] getSettableExpressionWriters( SettableStructObjectInspector objInspector) throws HiveException { List<? extends StructField> fieldsRef = objInspector.getAllStructFieldRefs(); VectorExpressionWriter[] writers = new VectorExpressionWriter[fieldsRef.size()]; for(int i=0; i<writers.length; ++i) { StructField fieldRef = fieldsRef.get(i); VectorExpressionWriter baseWriter = genVectorExpressionWritable( fieldRef.getFieldObjectInspector()); writers[i] = genVectorExpressionWritable(objInspector, fieldRef, baseWriter); } return writers; }
@Override public Object get(int fieldNum) { try { StructField fref = soi.getAllStructFieldRefs().get(fieldNum); return HCatRecordSerDe.serializeField( soi.getStructFieldData(wrappedObject, fref), fref.getFieldObjectInspector()); } catch (SerDeException e) { throw new IllegalStateException("SerDe Exception deserializing",e); } }
/** * Compiles the appropriate vector expression writers based on a struct object * inspector. */ public static VectorExpressionWriter[] genVectorStructExpressionWritables( StructObjectInspector oi) throws HiveException { VectorExpressionWriter[] writers = new VectorExpressionWriter[oi.getAllStructFieldRefs().size()]; final List<? extends StructField> fields = oi.getAllStructFieldRefs(); int i = 0; for(StructField field : fields) { writers[i++] = genVectorExpressionWritable(field.getFieldObjectInspector()); } return writers; }
byte[] serializeKeyField(Object keyValue, StructField keyField, ColumnMapping keyMapping) throws IOException { if (keyValue == null) { throw new IOException("HBase row key cannot be NULL"); } ObjectInspector keyFieldOI = keyField.getFieldObjectInspector(); if (!keyFieldOI.getCategory().equals(ObjectInspector.Category.PRIMITIVE) && keyMapping.isCategory(ObjectInspector.Category.PRIMITIVE)) { // we always serialize the String type using the escaped algorithm for LazyString return serialize(SerDeUtils.getJSONString(keyValue, keyFieldOI), PrimitiveObjectInspectorFactory.javaStringObjectInspector, 1, false); } // use the serialization option switch to write primitive values as either a variable // length UTF8 string or a fixed width bytes if serializing in binary format boolean writeBinary = keyMapping.binaryStorage.get(0); return serialize(keyValue, keyFieldOI, 1, writeBinary); }
return new DeserializerOrcWriter(serDe, sourceOi, allocSize); for (StructField sf : sourceOi.getAllStructFieldRefs()) { Category c = sf.getFieldObjectInspector().getCategory(); if (c != Category.PRIMITIVE) { LlapIoImpl.LOG.info("Not using VertorDeserializeOrcWriter: " + c + " is not supported");
public static boolean hasComplexObjects(LazyBinaryStructObjectInspector lazyBinaryStructObjectInspector) { List<? extends StructField> fields = lazyBinaryStructObjectInspector.getAllStructFieldRefs(); for (StructField field : fields) { if (field.getFieldObjectInspector().getCategory() != Category.PRIMITIVE) { return true; } } return false; }
public MyField(int fieldID, StructField field) { this.fieldID = fieldID; this.fieldName = field.getFieldName().toLowerCase(); this.fieldObjectInspector = field.getFieldObjectInspector(); this.fieldComment = field.getFieldComment(); }
protected static ObjectInspector[] getObjectInspectorsForBucketedCols(List<Integer> bucketIds , StructObjectInspector recordObjInspector) throws SerializationError { ObjectInspector[] result = new ObjectInspector[bucketIds.size()]; for (int i = 0; i < bucketIds.size(); i++) { int bucketId = bucketIds.get(i); result[i] = recordObjInspector.getAllStructFieldRefs().get( bucketId ).getFieldObjectInspector(); } return result; }
public static VectorExpressionWriter[] getSettableExpressionWriters( SettableStructObjectInspector objInspector) throws HiveException { List<? extends StructField> fieldsRef = objInspector.getAllStructFieldRefs(); VectorExpressionWriter[] writers = new VectorExpressionWriter[fieldsRef.size()]; for(int i=0; i<writers.length; ++i) { StructField fieldRef = fieldsRef.get(i); VectorExpressionWriter baseWriter = genVectorExpressionWritable( fieldRef.getFieldObjectInspector()); writers[i] = genVectorExpressionWritable(objInspector, fieldRef, baseWriter); } return writers; }
private static SampleRec deserializeInner(Object row, StructObjectInspector inspector) { List<? extends StructField> fields = inspector.getAllStructFieldRefs(); WritableStringObjectInspector f0ins = (WritableStringObjectInspector) fields.get(0).getFieldObjectInspector(); WritableIntObjectInspector f1ins = (WritableIntObjectInspector) fields.get(1).getFieldObjectInspector(); WritableStringObjectInspector f2ins = (WritableStringObjectInspector) fields.get(2).getFieldObjectInspector(); String f0 = f0ins.getPrimitiveJavaObject(inspector.getStructFieldData(row, fields.get(0))); int f1 = f1ins.get(inspector.getStructFieldData(row, fields.get(1))); String f2 = f2ins.getPrimitiveJavaObject(inspector.getStructFieldData(row, fields.get(2))); return new SampleRec(f0, f1, f2); }
public static ArrayList<TypeInfo> typeInfosFromStructObjectInspector( StructObjectInspector structObjectInspector) { List<? extends StructField> fields = structObjectInspector.getAllStructFieldRefs(); ArrayList<TypeInfo> typeInfoList = new ArrayList<TypeInfo>(fields.size()); for(StructField field : fields) { TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString( field.getFieldObjectInspector().getTypeName()); typeInfoList.add(typeInfo); } return typeInfoList; }
private static void addInputColumnsToList(ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) { StructObjectInspector OI = shape.getOI(); for (StructField f : OI.getAllStructFieldRefs()) { fieldNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } }
ObjectInspector.Category oiCategory = oi.getCategory(); switch (oiCategory) { case PRIMITIVE: StructObjectInspector soi = (StructObjectInspector) oi; List<Object> convertedRow = new ArrayList<Object>(); for (StructField structField : soi.getAllStructFieldRefs()) { Object convertedFieldValue = convertValue( soi.getStructFieldData(val, structField), structField.getFieldObjectInspector()); convertedRow.add(convertedFieldValue);
public static boolean hasComplexObjects(LazyBinaryStructObjectInspector lazyBinaryStructObjectInspector) { List<? extends StructField> fields = lazyBinaryStructObjectInspector.getAllStructFieldRefs(); for (StructField field : fields) { if (field.getFieldObjectInspector().getCategory() != Category.PRIMITIVE) { return true; } } return false; }
/** * Create a new struct object inspector for the list of struct fields, first removing the * prefix from the field name. * @param fields * @param prefixToRemove * @return */ private static ObjectInspector createStructFromFields(List<StructField> fields, String prefixToRemove) { int prefixLength = prefixToRemove.length() + 1; // also remove the '.' after the prefix ArrayList<String> fieldNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); for (StructField field : fields) { fieldNames.add(field.getFieldName().substring(prefixLength)); fieldOIs.add(field.getFieldObjectInspector()); } return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
public StructObjectInspector initialize(StructObjectInspector argOIs) throws UDFArgumentException { List<? extends StructField> inputFields = argOIs.getAllStructFieldRefs(); ObjectInspector[] udtfInputOIs = new ObjectInspector[inputFields.size()]; for (int i = 0; i < inputFields.size(); i++) { udtfInputOIs[i] = inputFields.get(i).getFieldObjectInspector(); } return initialize(udtfInputOIs); }
private static ExprNodeConstantDesc toStructConstDesc(ColumnInfo colInfo, ObjectInspector inspector, List<? extends StructField> fields) { List<?> values = (List<?>)((ConstantObjectInspector) inspector).getWritableConstantValue(); List<Object> constant = new ArrayList<Object>(); for (int i = 0; i < values.size(); i++) { Object value = values.get(i); PrimitiveObjectInspector fieldPoi = (PrimitiveObjectInspector) fields.get(i).getFieldObjectInspector(); constant.add(fieldPoi.getPrimitiveJavaObject(value)); } ExprNodeConstantDesc constantExpr = new ExprNodeConstantDesc(colInfo.getType(), constant); constantExpr.setFoldedFromCol(colInfo.getInternalName()); return constantExpr; }