private ObjectInspector getOiFromSerDe() throws IOException { try { return sourceSerDe.getObjectInspector(); } catch (SerDeException e) { throw new IOException(e); } }
public static StructObjectInspector getTableObjectInspector(@SuppressWarnings("deprecation") Deserializer deserializer) { try { ObjectInspector inspector = deserializer.getObjectInspector(); checkArgument(inspector.getCategory() == Category.STRUCT, "expected STRUCT: %s", inspector.getCategory()); return (StructObjectInspector) inspector; } catch (SerDeException e) { throw new RuntimeException(e); } }
public StructField getField(String fld) { try { StructObjectInspector structObjectInspector = (StructObjectInspector) getDeserializer() .getObjectInspector(); return structObjectInspector.getStructFieldRef(fld); } catch (Exception e) { throw new RuntimeException(e); } }
private static StructObjectInspector getStructOI(final Deserializer deserializer) throws Exception { ObjectInspector oi = deserializer.getObjectInspector(); if (oi.getCategory() != ObjectInspector.Category.STRUCT) { throw new UnsupportedOperationException(String.format("%s category not supported", oi.getCategory())); } return (StructObjectInspector) oi; }
public ArrayList<StructField> getFields() { ArrayList<StructField> fields = new ArrayList<StructField>(); try { Deserializer decoder = getDeserializer(); // Expand out all the columns of the table StructObjectInspector structObjectInspector = (StructObjectInspector) decoder .getObjectInspector(); List<? extends StructField> fld_lst = structObjectInspector .getAllStructFieldRefs(); for (StructField field : fld_lst) { fields.add(field); } } catch (SerDeException e) { throw new RuntimeException(e); } return fields; }
public StructField getField(String fld) { try { StructObjectInspector structObjectInspector = (StructObjectInspector) getDeserializer() .getObjectInspector(); return structObjectInspector.getStructFieldRef(fld); } catch (Exception e) { throw new RuntimeException(e); } }
public ArrayList<StructField> getFields() { ArrayList<StructField> fields = new ArrayList<StructField>(); try { Deserializer decoder = getDeserializer(); // Expand out all the columns of the table StructObjectInspector structObjectInspector = (StructObjectInspector) decoder .getObjectInspector(); List<? extends StructField> fld_lst = structObjectInspector .getAllStructFieldRefs(); for (StructField field : fld_lst) { fields.add(field); } } catch (SerDeException e) { throw new RuntimeException(e); } return fields; }
public SerDeFileMetadata(Deserializer sourceSerDe) throws SerDeException { TypeDescription schema = OrcInputFormat.convertTypeInfo( TypeInfoUtils.getTypeInfoFromObjectInspector(sourceSerDe.getObjectInspector())); this.schema = schema; addTypesFromSchema(schema); }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); TableDesc tbl = this.getConf().getTbl(); try { Deserializer serde = tbl.getDeserializerClass().newInstance(); SerDeUtils.initializeSerDe(serde, hconf, tbl.getProperties(), null); this.outputObjInspector = serde.getObjectInspector(); } catch (Exception e) { LOG.error("Generating output obj inspector from dummy object error", e); e.printStackTrace(); } }
public SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, String columnType, JobConf jobConf) throws SerDeException { this.skipPruning.set(false); this.partKey = partKey; this.columnName = columnName; this.columnType = columnType; deserializer = ReflectionUtils.newInstance(table.getDeserializerClass(), null); deserializer.initialize(jobConf, table.getProperties()); ObjectInspector inspector = deserializer.getObjectInspector(); LOG.debug("Type of obj insp: " + inspector.getTypeName()); soi = (StructObjectInspector) inspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); if (fields.size() > 1) { LOG.error("expecting single field in input"); } field = fields.get(0); fieldInspector = ObjectInspectorUtils.getStandardObjectInspector(field.getFieldObjectInspector()); } }
SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, String columnType, JobConf jobConf) throws SerDeException { this.partKey = partKey; this.columnName = columnName; this.columnType = columnType; deserializer = ReflectionUtils.newInstance(table.getDeserializerClass(), null); deserializer.initialize(jobConf, table.getProperties()); ObjectInspector inspector = deserializer.getObjectInspector(); if (LOG.isDebugEnabled()) { LOG.debug("Type of obj insp: " + inspector.getTypeName()); } soi = (StructObjectInspector) inspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); assert(fields.size() > 1) : "expecting single field in input"; field = fields.get(0); fieldInspector = ObjectInspectorUtils.getStandardObjectInspector(field.getFieldObjectInspector()); } }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); TableDesc tbl = this.getConf().getTbl(); try { Deserializer serde = tbl.getDeserializerClass().newInstance(); SerDeUtils.initializeSerDe(serde, hconf, tbl.getProperties(), null); this.outputObjInspector = serde.getObjectInspector(); } catch (Exception e) { LOG.error("Generating output obj inspector from dummy object error", e); e.printStackTrace(); } }
@SuppressWarnings("deprecation") private KeyValueReader getKeyValueReader(Collection<KeyValueReader> keyValueReaders, AbstractMapOperator mapOp) throws Exception { List<KeyValueReader> kvReaderList = new ArrayList<KeyValueReader>(keyValueReaders); // this sets up the map operator contexts correctly mapOp.initializeContexts(); Deserializer deserializer = mapOp.getCurrentDeserializer(); // deserializer is null in case of VectorMapOperator KeyValueReader reader = new KeyValueInputMerger(kvReaderList, deserializer, new ObjectInspector[] { deserializer == null ? null : deserializer.getObjectInspector() }, mapOp .getConf() .getSortCols()); return reader; }
@SuppressWarnings("deprecation") private KeyValueReader getKeyValueReader(Collection<KeyValueReader> keyValueReaders, AbstractMapOperator mapOp) throws Exception { List<KeyValueReader> kvReaderList = new ArrayList<KeyValueReader>(keyValueReaders); // this sets up the map operator contexts correctly mapOp.initializeContexts(); Deserializer deserializer = mapOp.getCurrentDeserializer(); // deserializer is null in case of VectorMapOperator KeyValueReader reader = new KeyValueInputMerger(kvReaderList, deserializer, new ObjectInspector[] { deserializer == null ? null : deserializer.getObjectInspector() }, mapOp .getConf() .getSortCols()); return reader; }
static Object evalExprWithPart(ExprNodeDesc expr, Partition p, List<VirtualColumn> vcs) throws SemanticException { StructObjectInspector rowObjectInspector; Table tbl = p.getTable(); try { rowObjectInspector = (StructObjectInspector) tbl .getDeserializer().getObjectInspector(); } catch (SerDeException e) { throw new SemanticException(e); } try { return PartExprEvalUtils.evalExprWithPart(expr, p, vcs, rowObjectInspector); } catch (HiveException e) { throw new SemanticException(e); } }
static Object evalExprWithPart(ExprNodeDesc expr, Partition p, List<VirtualColumn> vcs) throws SemanticException { StructObjectInspector rowObjectInspector; Table tbl = p.getTable(); try { rowObjectInspector = (StructObjectInspector) tbl .getDeserializer().getObjectInspector(); } catch (SerDeException e) { throw new SemanticException(e); } try { return PartExprEvalUtils.evalExprWithPart(expr, p, vcs, rowObjectInspector); } catch (HiveException e) { throw new SemanticException(e); } }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); firstRow = true; statsMap.put(Counter.DESERIALIZE_ERRORS.toString(), deserialize_error_count); statsMap.put(Counter.SERIALIZE_ERRORS.toString(), serialize_error_count); try { this.hconf = hconf; scriptOutputDeserializer = conf.getScriptOutputInfo() .getDeserializerClass().newInstance(); SerDeUtils.initializeSerDe(scriptOutputDeserializer, hconf, conf.getScriptOutputInfo().getProperties(), null); scriptInputSerializer = (Serializer) conf.getScriptInputInfo() .getDeserializerClass().newInstance(); scriptInputSerializer.initialize(hconf, conf.getScriptInputInfo() .getProperties()); outputObjInspector = scriptOutputDeserializer.getObjectInspector(); } catch (Exception e) { throw new HiveException(ErrorMsg.SCRIPT_INIT_ERROR.getErrorCodedMsg(), e); } }
public void initEmptyInputChildren(List<Operator<?>> children, Configuration hconf) throws SerDeException, Exception { setChildOperators(children); Map<String, Configuration> tableNameToConf = cloneConfsForNestedColPruning(hconf); for (Operator<?> child : children) { TableScanOperator tsOp = (TableScanOperator) child; StructObjectInspector soi = null; PartitionDesc partDesc = conf.getAliasToPartnInfo().get(tsOp.getConf().getAlias()); Configuration newConf = tableNameToConf.get(partDesc.getTableDesc().getTableName()); Deserializer serde = partDesc.getTableDesc().getDeserializer(); partDesc.setProperties(partDesc.getProperties()); MapOpCtx opCtx = new MapOpCtx(tsOp.getConf().getAlias(), child, partDesc); StructObjectInspector tableRowOI = (StructObjectInspector) serde.getObjectInspector(); initObjectInspector(newConf, opCtx, tableRowOI); soi = opCtx.rowObjectInspector; child.getParentOperators().add(this); childrenOpToOI.put(child, soi); } }
public void init(Configuration hconf) throws Exception { VectorPartitionDesc vectorPartDesc = partDesc.getVectorPartitionDesc(); partDeserializer = partDesc.getDeserializer(hconf); if (partDeserializer instanceof OrcSerde) { // UNDONE: We need to get the table schema inspector from self-describing Input File // Formats like ORC. Modify the ORC serde instead? For now, this works. partRawRowObjectInspector = (StructObjectInspector) OrcStruct.createObjectInspector(tableStructTypeInfo); } else { partRawRowObjectInspector = (StructObjectInspector) partDeserializer.getObjectInspector(); } TypeInfo[] dataTypeInfos = vectorPartDesc.getDataTypeInfos(); vectorAssign = new VectorAssignRow(); // Initialize with data type conversion parameters. readerColumnCount = vectorAssign.initConversion(dataTypeInfos, tableRowTypeInfos, dataColumnsToIncludeTruncated); } }
V value = recordReader.createValue(); StructObjectInspector rowInspector = (StructObjectInspector) deserializer.getObjectInspector(); StructField field = rowInspector.getStructFieldRef("test");