@Override public AvroGenericRecordWritable getWritable(byte[] value) { GenericRecord avroRecord = null; try { avroRecord = dataReader.read(null, DecoderFactory.get().binaryDecoder(value, null)); } catch (IOException e) { Throwables.propagate(new SerDeException(e)); } avroGenericRecordWritable.setRecord(avroRecord); avroGenericRecordWritable.setRecordReaderID(uid); avroGenericRecordWritable.setFileSchema(avroRecord.getSchema()); return avroGenericRecordWritable; } }
GenericRecord r = recordWritable.getRecord(); Schema fileSchema = recordWritable.getFileSchema(); UID recordReaderId = recordWritable.getRecordReaderID();
@Override public Iterable<AvroGenericRecordWritable> convertRecord(Schema outputSchema, GenericRecord inputRecord, WorkUnitState workUnit) throws DataConversionException { AvroGenericRecordWritable avroWritable = new AvroGenericRecordWritable(); avroWritable.setRecord(inputRecord); avroWritable.setFileSchema(outputSchema); avroWritable.setRecordReaderID(this.uid); return Lists.newArrayList(avroWritable); } }
value.setFileSchema(wrapper.getFileSchema()); value.setRecord(wrapper.getRecord()); long decodeTime = ((secondTime - tempTime));
public Writable serialize(Object o, ObjectInspector objectInspector, List<String> columnNames, List<TypeInfo> columnTypes, Schema schema) throws AvroSerdeException { StructObjectInspector soi = (StructObjectInspector) objectInspector; GenericData.Record record = new GenericData.Record(schema); List<? extends StructField> outputFieldRefs = soi.getAllStructFieldRefs(); if(outputFieldRefs.size() != columnNames.size()) { throw new AvroSerdeException("Number of input columns was different than output columns (in = " + columnNames.size() + " vs out = " + outputFieldRefs.size()); } int size = schema.getFields().size(); if(outputFieldRefs.size() != size) { throw new AvroSerdeException("Hive passed in a different number of fields than the schema expected: (Hive wanted " + outputFieldRefs.size() +", Avro expected " + schema.getFields().size()); } List<? extends StructField> allStructFieldRefs = soi.getAllStructFieldRefs(); List<Object> structFieldsDataAsList = soi.getStructFieldsDataAsList(o); for(int i = 0; i < size; i++) { Field field = schema.getFields().get(i); TypeInfo typeInfo = columnTypes.get(i); StructField structFieldRef = allStructFieldRefs.get(i); Object structFieldData = structFieldsDataAsList.get(i); ObjectInspector fieldOI = structFieldRef.getFieldObjectInspector(); Object val = serialize(typeInfo, fieldOI, structFieldData, field.schema()); record.put(field.name(), val); } if(!GenericData.get().validate(schema, record)) { throw new SerializeToAvroException(schema, record); } cache.setRecord(record); return cache; }
int offset = 0; AvroGenericRecordWritable avroWritable = new AvroGenericRecordWritable(); avroWritable.readFields(data, offset, data.length, ws, rs); } catch (IOException ioe) { throw new AvroObjectInspectorException("Error deserializing avro payload", ioe); avroWritable.readFields(data, ws, rs); } catch (IOException ioe) { throw new AvroObjectInspectorException("Error deserializing avro payload", ioe);
@Override public void write(Writable writable) throws IOException { if(!(writable instanceof AvroGenericRecordWritable)) { throw new IOException("Expecting instance of AvroGenericRecordWritable, " + "but received" + writable.getClass().getCanonicalName()); } AvroGenericRecordWritable r = (AvroGenericRecordWritable)writable; dfw.append(r.getRecord()); }
public AvroGenericRecordWritable decode(byte[] payload) { try { MessageDecoderHelper helper = new MessageDecoderHelper(registry, topicName, payload).invoke(); DatumReader<Record> reader = new GenericDatumReader<Record>(helper.getTargetSchema()); log.debug("Trying to read kafka payload"); log.debug("buffer: " + helper.getBuffer()); log.debug("start: " + helper.getStart()); log.debug("length: " + helper.getLength()); log.debug("target schema: " + helper.getTargetSchema()); log.debug("schema: " + helper.getSchema()); GenericRecord record = reader.read(null, decoderFactory.binaryDecoder(helper.getBuffer().array(), helper.getStart(), helper.getLength(), null)); log.debug("Read kafka payload as " + record); AvroGenericRecordWritable grw = new AvroGenericRecordWritable(record); grw.setFileSchema(latestSchema); return grw; } catch (IOException e) { throw new MessageDecoderException(e); } }
@Override public AvroGenericRecordWritable createValue() { return new AvroGenericRecordWritable(); }
public Writable serialize(Object o, ObjectInspector objectInspector, List<String> columnNames, List<TypeInfo> columnTypes, Schema schema) throws AvroSerdeException { StructObjectInspector soi = (StructObjectInspector) objectInspector; GenericData.Record record = new GenericData.Record(schema); List<? extends StructField> outputFieldRefs = soi.getAllStructFieldRefs(); if(outputFieldRefs.size() != columnNames.size()) throw new AvroSerdeException("Number of input columns was different than output columns (in = " + columnNames.size() + " vs out = " + outputFieldRefs.size()); int size = schema.getFields().size(); if(outputFieldRefs.size() != size) // Hive does this check for us, so we should be ok. throw new AvroSerdeException("Hive passed in a different number of fields than the schema expected: (Hive wanted " + outputFieldRefs.size() +", Avro expected " + schema.getFields().size()); List<? extends StructField> allStructFieldRefs = soi.getAllStructFieldRefs(); List<Object> structFieldsDataAsList = soi.getStructFieldsDataAsList(o); for(int i = 0; i < size; i++) { Field field = schema.getFields().get(i); TypeInfo typeInfo = columnTypes.get(i); StructField structFieldRef = allStructFieldRefs.get(i); Object structFieldData = structFieldsDataAsList.get(i); ObjectInspector fieldOI = structFieldRef.getFieldObjectInspector(); Object val = serialize(typeInfo, fieldOI, structFieldData, field.schema()); record.put(field.name(), val); } if(!GenericData.get().validate(schema, record)) throw new SerializeToAvroException(schema, record); cache.setRecord(record); return cache; }
int offset = 0; AvroGenericRecordWritable avroWritable = new AvroGenericRecordWritable(); avroWritable.readFields(data, offset, data.length, ws, rs); } catch (IOException ioe) { throw new AvroObjectInspectorException("Error deserializing avro payload", ioe); avroWritable.readFields(data, ws, rs); } catch (IOException ioe) { throw new AvroObjectInspectorException("Error deserializing avro payload", ioe);
@Override public void write(Writable writable) throws IOException { if(!(writable instanceof AvroGenericRecordWritable)) { throw new IOException("Expecting instance of AvroGenericRecordWritable, " + "but received" + writable.getClass().getCanonicalName()); } AvroGenericRecordWritable r = (AvroGenericRecordWritable)writable; dfw.append(r.getRecord()); }
@Override public AvroGenericRecordWritable createValue() { return new AvroGenericRecordWritable(); }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(isEmptyInput || !reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); record.setRecordReaderID(recordReaderID); record.setFileSchema(reader.getSchema()); return true; }
@Override public Iterable<AvroGenericRecordWritable> convertRecord(Schema outputSchema, GenericRecord inputRecord, WorkUnitState workUnit) throws DataConversionException { AvroGenericRecordWritable avroWritable = new AvroGenericRecordWritable(); avroWritable.setRecord(inputRecord); avroWritable.setFileSchema(outputSchema); avroWritable.setRecordReaderID(this.uid); return Lists.newArrayList(avroWritable); } }
GenericRecord r = recordWritable.getRecord(); Schema fileSchema = recordWritable.getFileSchema(); UID recordReaderId = recordWritable.getRecordReaderID();
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(!reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); return true; }
int offset = 0; AvroGenericRecordWritable avroWritable = new AvroGenericRecordWritable(); avroWritable.readFields(data, offset, data.length, ws, rs); } catch (IOException ioe) { throw new AvroObjectInspectorException("Error deserializing avro payload", ioe); avroWritable.readFields(data, ws, rs); } catch (IOException ioe) { throw new AvroObjectInspectorException("Error deserializing avro payload", ioe);
@Override public byte[] getBytes(AvroGenericRecordWritable writable) { GenericRecord record = writable.getRecord(); byte[] valueBytes = null; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { BinaryEncoder be = EncoderFactory.get().directBinaryEncoder(out, null); gdw.setSchema(record.getSchema()); gdw.write(record, be); out.flush(); valueBytes = out.toByteArray(); } catch (IOException e) { Throwables.propagate(new SerDeException(e)); } return valueBytes; }
@Override public AvroGenericRecordWritable createValue() { return new AvroGenericRecordWritable(); }