public Integer getInt(String key, int iDef) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; WritableComparable fieldValue = orcStruct.getFieldValue(key); if (fieldValue != null) { Integer value = ((IntWritable) fieldValue).get(); return value; } } return iDef; }
@Override public V createValue() { return (V) OrcStruct.createValue(schema); }
static OrcStruct nextStruct(ColumnVector vector, int row, TypeDescription schema, Object previous) { if (vector.isRepeating) { row = 0; } if (vector.noNulls || !vector.isNull[row]) { OrcStruct result; List<TypeDescription> childrenTypes = schema.getChildren(); int numChildren = childrenTypes.size(); if (previous == null || previous.getClass() != OrcStruct.class) { result = new OrcStruct(schema); } else { result = (OrcStruct) previous; } StructColumnVector struct = (StructColumnVector) vector; for(int f=0; f < numChildren; ++f) { result.setFieldValue(f, nextValue(struct.fields[f], row, childrenTypes.get(f), result.getFieldValue(f))); } return result; } else { return null; } }
private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); if (this.struct == null) { this.struct = new OrcStruct(TypeDescription.fromString(this.typeDescription)); } this.struct.readFields(in); } }
static void setStructValue(TypeDescription schema, StructColumnVector vector, int row, OrcStruct value) { List<TypeDescription> children = schema.getChildren(); for(int c=0; c < value.getNumFields(); ++c) { setColumn(children.get(c), vector.fields[c], row, value.getFieldValue(c)); } }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!ensureBatch()) { return false; } if (schema.getCategory() == TypeDescription.Category.STRUCT) { OrcStruct result = (OrcStruct) row; List<TypeDescription> children = schema.getChildren(); int numberOfChildren = children.size(); for(int i=0; i < numberOfChildren; ++i) { result.setFieldValue(i, OrcMapredRecordReader.nextValue(batch.cols[i], rowInBatch, children.get(i), result.getFieldValue(i))); } } else { OrcMapredRecordReader.nextValue(batch.cols[0], rowInBatch, schema, row); } rowInBatch += 1; return true; }
return new HiveDecimalWritable(); case STRUCT: { OrcStruct result = new OrcStruct(type); int c = 0; for(TypeDescription child: type.getChildren()) { result.setFieldValue(c++, createValue(child));
public OrcStruct serialize(BaseRow row, OrcStruct struct) { for (int i = 0; i < fieldConverters.length; i++) { if (row.isNullAt(i)) { struct.setFieldValue(i, null); } else { struct.setFieldValue(i, fieldConverters[i].convert(BaseRowUtil.get(row, i, fieldTypes[i]))); } } return struct; }
/** * {@inheritDoc} */ protected void map(final CombineFileWritable key, final OrcStruct value, final Context context) throws IOException, InterruptedException { if (value != null && value.toString() != null && value.toString().isEmpty()) { return; } final OrcValue orcValue = new OrcValue(); orcValue.value = value; final Text mapperKey = baseMapper.getKey(key.getFileName()); context.write(mapperKey, orcValue); } }
@Override public boolean next(NullWritable key, V value) throws IOException { if (!ensureBatch()) { return false; } if (schema.getCategory() == TypeDescription.Category.STRUCT) { OrcStruct result = (OrcStruct) value; List<TypeDescription> children = schema.getChildren(); int numberOfChildren = children.size(); for(int i=0; i < numberOfChildren; ++i) { result.setFieldValue(i, nextValue(batch.cols[i], rowInBatch, children.get(i), result.getFieldValue(i))); } } else { nextValue(batch.cols[0], rowInBatch, schema, value); } rowInBatch += 1; return true; }
@Override public Long getLong(String key, Long lDef) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; WritableComparable fieldValue = orcStruct.getFieldValue(key); if (fieldValue != null) { Long value = ((LongWritable) fieldValue).get(); return value; } } return lDef; }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (key == null || key.fileName == null) { key = new CombineFileWritable(); key.fileName = path.toString(); } if (!ensureBatch()) { return false; } if (schema.getCategory() == TypeDescription.Category.STRUCT) { OrcStruct result = row; List<TypeDescription> children = schema.getChildren(); int numberOfChildren = children.size(); for (int i = 0; i < numberOfChildren; ++i) { result.setFieldValue(i, OrcMapredRecordReader.nextValue(batch.cols[i], rowInBatch, children.get(i), result.getFieldValue(i))); } } else { OrcMapredRecordReader.nextValue(batch.cols[0], rowInBatch, schema, row); } rowInBatch += 1; return true; }
public OrcMapreduceRecordReader(RecordReader reader, TypeDescription schema) throws IOException { this.batchReader = reader; this.batch = schema.createRowBatch(); this.schema = schema; rowInBatch = 0; this.row = (V) OrcStruct.createValue(schema); }
@Override public String getString(String key, String strDef) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; WritableComparable fieldValue = orcStruct.getFieldValue(key); if (fieldValue != null) { String value = ((Text) fieldValue).toString(); return value; } } return strDef; }
@Override public void readFields(DataInput input) throws IOException { byte oldTag = tag; tag = input.readByte(); if (input.readBoolean()) { if (oldTag != tag || object == null) { object = OrcStruct.createValue(schema.getChildren().get(tag)); } object.readFields(input); } else { object = null; } }
public Float getFloat(String key, float fDef) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; WritableComparable fieldValue = orcStruct.getFieldValue(key); if (fieldValue != null) { Float value = ((FloatWritable) fieldValue).get(); return value; } } return fDef; }
@Override public void readFields(DataInput input) throws IOException { for(int f=0; f < fields.length; ++f) { if (input.readBoolean()) { if (fields[f] == null) { fields[f] = createValue(schema.getChildren().get(f)); } fields[f].readFields(input); } else { fields[f] = null; } } }
@Override public Double getDouble(String key, Double dDef) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; WritableComparable fieldValue = orcStruct.getFieldValue(key); if (fieldValue != null) { Double value = ((DoubleWritable) fieldValue).get(); return value; } } return dDef; }
@Override public void configure(JobConf conf) { if (value == null) { TypeDescription schema = TypeDescription.fromString(OrcConf.MAPRED_SHUFFLE_VALUE_SCHEMA .getString(conf)); value = OrcStruct.createValue(schema); } } }
@Override public Boolean getBoolean(String key, Boolean bDef) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; WritableComparable fieldValue = orcStruct.getFieldValue(key); if (fieldValue != null) { Boolean value = ((BooleanWritable) fieldValue).get(); return value; } } return bDef; }