@Override public StructField getStructFieldRef(String fieldName) { return wrapped.getStructFieldRef(fieldName); }
@Override public StructField getStructFieldRef(String fieldName) { return wrapped.getStructFieldRef(fieldName); }
public StructField getField(String fld) { try { StructObjectInspector structObjectInspector = (StructObjectInspector) getDeserializer() .getObjectInspector(); return structObjectInspector.getStructFieldRef(fld); } catch (Exception e) { throw new RuntimeException(e); } }
private StructField getStructField(StructObjectInspector oi, String name) { if (hiveColIndexParsing) { int colIndex = getColIndex(name); if (colIndex >= 0) { return oi.getAllStructFieldRefs().get(colIndex); } } // FIXME: linear scan inside the below method...get a map here or something.. return oi.getStructFieldRef(name); }
public StructField getField(String fld) { try { StructObjectInspector structObjectInspector = (StructObjectInspector) getDeserializer() .getObjectInspector(); return structObjectInspector.getStructFieldRef(fld); } catch (Exception e) { throw new RuntimeException(e); } }
/** * Builds the assigners from an object inspector and from a list of columns. * @param outputBatch The batch to which the assigners are bound * @param outputOI The row object inspector * @param columnMap Vector column map * @param outputColumnNames Column names, used both to find the vector columns and the * @return * @throws HiveException */ public static VectorColumnAssign[] buildAssigners(VectorizedRowBatch outputBatch, ObjectInspector outputOI, Map<String, Integer> columnMap, List<String> outputColumnNames) throws HiveException { StructObjectInspector soi = (StructObjectInspector) outputOI; VectorColumnAssign[] vcas = new VectorColumnAssign[outputColumnNames.size()]; for (int i=0; i<outputColumnNames.size(); ++i) { String columnName = outputColumnNames.get(i); Integer columnIndex = columnMap.get(columnName); StructField columnRef = soi.getStructFieldRef(columnName); ObjectInspector valueOI = columnRef.getFieldObjectInspector(); vcas[i] = buildObjectAssign(outputBatch, columnIndex, valueOI); } return vcas; }
protected void preparePartitioningFields() { final int numPartitions = table.getPartitionKeys().size(); this.partitionFieldData = new Object[numPartitions]; this.partitionObjInspectors = new ObjectInspector[numPartitions]; int startIdx = inputRowObjectInspector.getAllStructFieldRefs().size() - numPartitions; int endIdx = inputRowObjectInspector.getAllStructFieldRefs().size(); int j = 0; for (int i = startIdx; i < endIdx; i++) { StructField structField = inputRowObjectInspector.getAllStructFieldRefs().get(i); partitionObjInspectors[j++] = structField.getFieldObjectInspector(); } this.partitionStructFields = new StructField[partitionColumns.size()]; for (int i = 0; i < partitionColumns.size(); i++) { String partCol = partitionColumns.get(i); partitionStructFields[i] = inputRowObjectInspector.getStructFieldRef(partCol); } }
/** * Builds the assigners from an object inspector and from a list of columns. * @param outputBatch The batch to which the assigners are bound * @param outputOI The row object inspector * @param columnMap Vector column map * @param outputColumnNames Column names, used both to find the vector columns and the * @return * @throws HiveException */ public static VectorColumnAssign[] buildAssigners(VectorizedRowBatch outputBatch, ObjectInspector outputOI, Map<String, Integer> columnMap, List<String> outputColumnNames) throws HiveException { StructObjectInspector soi = (StructObjectInspector) outputOI; VectorColumnAssign[] vcas = new VectorColumnAssign[outputColumnNames.size()]; for (int i=0; i<outputColumnNames.size(); ++i) { String columnName = outputColumnNames.get(i); Integer columnIndex = columnMap.get(columnName); StructField columnRef = soi.getStructFieldRef(columnName); ObjectInspector valueOI = columnRef.getFieldObjectInspector(); vcas[i] = buildObjectAssign(outputBatch, columnIndex, valueOI); } return vcas; }
public KeyValueInputMerger(List<KeyValueReader> multiMRInputs, Deserializer deserializer, ObjectInspector[] inputObjInspectors, List<String> sortCols) throws Exception { //get KeyValuesReaders from the LogicalInput and add them to priority queue int initialCapacity = multiMRInputs.size(); pQueue = new PriorityQueue<KeyValueReader>(initialCapacity, new KVReaderComparator()); this.inputObjInspectors = inputObjInspectors; this.deserializer = deserializer; fieldOIs = new ArrayList<ObjectInspector>(); structFields = new ArrayList<StructField>(); StructObjectInspector structOI = (StructObjectInspector) inputObjInspectors[0]; for (String field : sortCols) { StructField sf = structOI.getStructFieldRef(field); structFields.add(sf); ObjectInspector stdOI = ObjectInspectorUtils.getStandardObjectInspector(sf.getFieldObjectInspector()); fieldOIs.add(stdOI); } l4j.info("Initialized the priority queue with multi mr inputs: " + multiMRInputs.size()); for (KeyValueReader input : multiMRInputs) { addToQueue(input); } }
hiveTypes[i] = column.getHiveType(); StructField field = rowInspector.getStructFieldRef(column.getName()); structFields[i] = field; fieldInspectors[i] = field.getFieldObjectInspector();
StructObjectInspector structOI = (StructObjectInspector) inputObjInspectors[0]; for (String field : sortCols) { StructField sf = structOI.getStructFieldRef(field); structFields.add(sf); ObjectInspector stdOI =
/** * @param kafkaWritable Kafka writable object containing the row plus kafka metadata * @param rowBoat Boat sized to width of the kafka row plus metadata to carry the row to operator upstream. * * @throws SerDeException in case of any serde issue. */ void deserializeKWritable(KafkaWritable kafkaWritable, Object[] rowBoat) throws SerDeException { final Object row = delegateSerDe.deserialize(bytesConverter.getWritable(kafkaWritable.getValue())); //first add the value payload elements for (int i = 0; i < metadataStartIndex; i++) { rowBoat[i] = delegateDeserializerOI.getStructFieldData(row, delegateDeserializerOI.getStructFieldRef(columnNames.get(i))); } //add the metadata columns for (int i = metadataStartIndex; i < columnNames.size(); i++) { final MetadataColumn metadataColumn = MetadataColumn.forName(columnNames.get(i)); rowBoat[i] = (kafkaWritable.getHiveWritable(metadataColumn)); } }
@SuppressWarnings("unchecked") private List<Object> mergeJoinComputeKeys(Object row, Byte alias) throws HiveException { if ((joinKeysObjectInspectors != null) && (joinKeysObjectInspectors[alias] != null)) { return JoinUtil.computeKeys(row, joinKeys[alias], joinKeysObjectInspectors[alias]); } else { row = ObjectInspectorUtils.copyToStandardObject(row, inputObjInspectors[alias], ObjectInspectorCopyOption.WRITABLE); StructObjectInspector soi = (StructObjectInspector) inputObjInspectors[alias]; StructField sf = soi.getStructFieldRef(Utilities.ReduceField.KEY.toString()); return (List<Object>) soi.getStructFieldData(row, sf); } }
private static void assertFileContentsDwrfHive( Type type, TempFile tempFile, Iterable<?> expectedValues) throws Exception { JobConf configuration = new JobConf(new Configuration(false)); configuration.set(READ_COLUMN_IDS_CONF_STR, "0"); configuration.setBoolean(READ_ALL_COLUMNS, false); Path path = new Path(tempFile.getFile().getAbsolutePath()); com.facebook.hive.orc.Reader reader = com.facebook.hive.orc.OrcFile.createReader( path.getFileSystem(configuration), path, configuration); boolean[] include = new boolean[reader.getTypes().size() + 100000]; Arrays.fill(include, true); com.facebook.hive.orc.RecordReader recordReader = reader.rows(include); StructObjectInspector rowInspector = (StructObjectInspector) reader.getObjectInspector(); StructField field = rowInspector.getStructFieldRef("test"); Iterator<?> iterator = expectedValues.iterator(); Object rowData = null; while (recordReader.hasNext()) { rowData = recordReader.next(rowData); Object expectedValue = iterator.next(); Object actualValue = rowInspector.getStructFieldData(rowData, field); actualValue = decodeRecordReaderValue(type, actualValue); assertColumnValueEquals(type, actualValue, expectedValue); } assertFalse(iterator.hasNext()); }
private static void assertFileContentsOrcHive( Type type, TempFile tempFile, Iterable<?> expectedValues) throws Exception { JobConf configuration = new JobConf(new Configuration(false)); configuration.set(READ_COLUMN_IDS_CONF_STR, "0"); configuration.setBoolean(READ_ALL_COLUMNS, false); Reader reader = OrcFile.createReader( new Path(tempFile.getFile().getAbsolutePath()), new ReaderOptions(configuration)); org.apache.hadoop.hive.ql.io.orc.RecordReader recordReader = reader.rows(); StructObjectInspector rowInspector = (StructObjectInspector) reader.getObjectInspector(); StructField field = rowInspector.getStructFieldRef("test"); Iterator<?> iterator = expectedValues.iterator(); Object rowData = null; while (recordReader.hasNext()) { rowData = recordReader.next(rowData); Object expectedValue = iterator.next(); Object actualValue = rowInspector.getStructFieldData(rowData, field); actualValue = decodeRecordReaderValue(type, actualValue); assertColumnValueEquals(type, actualValue, expectedValue); } assertFalse(iterator.hasNext()); }
StructField field = rowInspector.getStructFieldRef("test");
public void testParquetHiveSerDeComplexTypes() throws Throwable { // Initialize ParquetHiveSerDe serDe = new ParquetHiveSerDe(); Configuration conf = new Configuration(); Properties tblProperties = new Properties(); tblProperties.setProperty(serdeConstants.LIST_COLUMNS, "a,s"); tblProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES, "int,struct<a:int,b:string>"); conf.set(ColumnProjectionUtils.READ_NESTED_COLUMN_PATH_CONF_STR, "s.b"); serDe.initialize(conf, tblProperties); // Generate test data Writable[] wb = new Writable[1]; wb[0] = new BytesWritable("foo".getBytes("UTF-8")); Writable[] ws = new Writable[2]; ws[0] = null; ArrayWritable awb = new ArrayWritable(Writable.class, wb); ws[1] = awb; ArrayWritable aws = new ArrayWritable(Writable.class, ws); // Inspect the test data StructObjectInspector soi = (StructObjectInspector) serDe.getObjectInspector(); StructField s = soi.getStructFieldRef("s"); assertEquals(awb, soi.getStructFieldData(aws, s)); StructObjectInspector boi = (StructObjectInspector) s.getFieldObjectInspector(); StructField b = boi.getStructFieldRef("b"); assertEquals(wb[0], boi.getStructFieldData(awb, b)); }
@Override public ObjectInspector initialize(ObjectInspector rowInspector) throws HiveException { leftInspector = leftEvaluator.initialize(rowInspector); if (expr.getIsList()) { structObjectInspector = (StructObjectInspector) ((ListObjectInspector) leftInspector) .getListElementObjectInspector(); } else { structObjectInspector = (StructObjectInspector) leftInspector; } field = structObjectInspector.getStructFieldRef(expr.getFieldName()); structFieldObjectInspector = field.getFieldObjectInspector(); if (expr.getIsList()) { resultObjectInspector = ObjectInspectorFactory .getStandardListObjectInspector(structFieldObjectInspector); } else { resultObjectInspector = structFieldObjectInspector; } return outputOI = resultObjectInspector; }
@Override public ObjectInspector initialize(ObjectInspector rowInspector) throws HiveException { leftInspector = leftEvaluator.initialize(rowInspector); if (expr.getIsList()) { structObjectInspector = (StructObjectInspector) ((ListObjectInspector) leftInspector) .getListElementObjectInspector(); } else { structObjectInspector = (StructObjectInspector) leftInspector; } field = structObjectInspector.getStructFieldRef(expr.getFieldName()); structFieldObjectInspector = field.getFieldObjectInspector(); if (expr.getIsList()) { resultObjectInspector = ObjectInspectorFactory .getStandardListObjectInspector(structFieldObjectInspector); } else { resultObjectInspector = structFieldObjectInspector; } return outputOI = resultObjectInspector; }
List<? extends StructField> fields = readerInspector.getAllStructFieldRefs(); HiveDecimalObjectInspector doi = (HiveDecimalObjectInspector) readerInspector. getStructFieldRef("dec").getFieldObjectInspector(); RecordReader rows = reader.rows(); while (rows.hasNext()) {