@Override public Object deserialize(Writable writable) throws SerDeException { if(badSchema) { throw new BadSchemaException(); } return avroDeserializer.deserialize(columnNames, columnTypes, writable, schema); }
@Override public Writable serialize(Object o, ObjectInspector objectInspector) throws SerDeException { if(badSchema) { throw new BadSchemaException(); } return avroSerializer.serialize(o, objectInspector, columnNames, columnTypes, schema); }
/** * Generate a list of of TypeInfos from an Avro schema. This method is * currently public due to some weirdness in deserializing unions, but * will be made private once that is resolved. * @param schema Schema to generate field types for * @return List of TypeInfos, each element of which is a TypeInfo derived * from the schema. * @throws AvroSerdeException for problems during conversion. */ public static List<TypeInfo> generateColumnTypes(Schema schema) throws AvroSerdeException { return generateColumnTypes (schema, null); }
private void setColumnsIfPossible(WorkUnitState state) throws SerDeException { AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator( AvroSerdeUtils.determineSchemaOrReturnErrorSchema(state.getProperties())); List<String> columnNames = aoig.getColumnNames(); List<TypeInfo> columnTypes = aoig.getColumnTypes(); state.setProp(IOConstants.COLUMNS, StringUtils.join(columnNames, ",")); state.setProp(IOConstants.COLUMNS_TYPES, StringUtils.join(columnTypes, ",")); }
public AvroObjectInspectorGenerator(Schema schema) throws SerDeException { verifySchemaIsARecord(schema); this.columnNames = AvroObjectInspectorGenerator.generateColumnNames(schema); this.columnTypes = SchemaToTypeInfo.generateColumnTypes(schema); this.columnComments = AvroObjectInspectorGenerator.generateColumnComments(schema); assert columnNames.size() == columnTypes.size(); this.oi = createObjectInspector(); }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(isEmptyInput || !reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); record.setRecordReaderID(recordReaderID); record.setFileSchema(reader.getSchema()); return true; }
private Properties createPropertiesForHiveAvroForwardEvolvedSchema() { Properties tbl = new Properties(); tbl.setProperty("cola.avro.serialization.type", "avro"); tbl.setProperty("cola.avro." + AvroTableProperties.SCHEMA_LITERAL.getPropName(), RECORD_SCHEMA_EVOLVED); tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, ":key,cola:avro"); tbl.setProperty(HBaseSerDe.HBASE_AUTOGENERATE_STRUCT, "true"); return tbl; }
@Override public AvroGenericRecordWritable createValue() { return new AvroGenericRecordWritable(); }
private Object serializeEnum(TypeInfo typeInfo, PrimitiveObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException { return enums.retrieve(schema).retrieve(serializePrimitive(typeInfo, fieldOI, structFieldData, schema)); }
@Override public Schema determineSchemaOrReturnErrorSchema(Configuration conf, Properties props) { // AvroSerDe does not propagate initialization exceptions. Instead, it stores just an exception's message in // this.configErrors (see https://issues.apache.org/jira/browse/HIVE-7868). In Presto, such behavior is not // at all useful, as silenced exception usually carries important information which may be otherwise unavailable. try { return AvroSerdeUtils.determineSchemaOrThrowException(conf, props); } catch (IOException | AvroSerdeException e) { throw new RuntimeException(e); } } }
/** * Retrieve (or create if it doesn't exist) the correct Instance for this * SeedObject */ public Instance retrieve(SeedObject hv) throws AvroSerdeException { return retrieve(hv, null); }
@Override public void initialize(Configuration configuration, Properties tableProperties, Properties partitionProperties) throws SerDeException { // Avro should always use the table properties for initialization (see HIVE-6835). initialize(configuration, tableProperties); }
@Override protected InstanceCache<Object, GenericEnumSymbol> makeInstance(final Schema schema, Set<Schema> seenSchemas) { return new InstanceCache<Object, GenericEnumSymbol>() { @Override protected GenericEnumSymbol makeInstance(Object seed, Set<Object> seenSchemas) { return new GenericData.EnumSymbol(schema, seed.toString()); } }; } };
public static Buffer getBufferFromDecimal(HiveDecimal dec, int scale) { if (dec == null) { return null; } // NOTE: Previously, we did OldHiveDecimal.setScale(scale), called OldHiveDecimal // unscaledValue().toByteArray(). return AvroSerdeUtils.getBufferFromBytes(dec.bigIntegerBytesScaled(scale)); }
@Override public boolean shouldStoreFieldsInMetastore(Map<String, String> tableParams) { return !hasExternalSchema(tableParams); } }
private Properties createPropertiesForHiveAvroSchemaUrl(String schemaUrl) { Properties tbl = new Properties(); tbl.setProperty("cola.avro.serialization.type", "avro"); tbl.setProperty("cola.avro." + AvroTableProperties.SCHEMA_URL.getPropName(), schemaUrl); tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, ":key,cola:avro"); tbl.setProperty(HBaseSerDe.HBASE_AUTOGENERATE_STRUCT, "true"); return tbl; }
@Override public AvroGenericRecordWritable createValue() { return new AvroGenericRecordWritable(); }
private Properties createPropertiesForHiveAvroBackwardEvolvedSchema() { Properties tbl = new Properties(); tbl.setProperty("cola.avro.serialization.type", "avro"); tbl.setProperty("cola.avro." + AvroTableProperties.SCHEMA_LITERAL.getPropName(), RECORD_SCHEMA); tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, ":key,cola:avro"); tbl.setProperty(HBaseSerDe.HBASE_AUTOGENERATE_STRUCT, "true"); return tbl; }
private Properties createPropertiesForHiveAvroSchemaInline() { Properties tbl = new Properties(); tbl.setProperty("cola.avro.serialization.type", "avro"); tbl.setProperty("cola.avro." + AvroTableProperties.SCHEMA_LITERAL.getPropName(), RECORD_SCHEMA); tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, ":key,cola:avro"); tbl.setProperty(HBaseSerDe.HBASE_AUTOGENERATE_STRUCT, "true"); return tbl; }
private Properties createPropertiesForHiveAvroExternalSchema() { Properties tbl = new Properties(); tbl.setProperty("cola.avro.serialization.type", "avro"); tbl.setProperty(AvroTableProperties.SCHEMA_RETRIEVER.getPropName(), "org.apache.hadoop.hive.hbase.HBaseTestAvroSchemaRetriever"); tbl.setProperty("cola.avro." + serdeConstants.SERIALIZATION_CLASS, "org.apache.hadoop.hive.hbase.avro.Employee"); tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, ":key,cola:avro"); tbl.setProperty(HBaseSerDe.HBASE_AUTOGENERATE_STRUCT, "true"); return tbl; }