/** * Serializes the field object using the datumWriter. * * @param <T> class type of object to be serialized. * @param datumWriter AVRO datum writer for given schema. * @param object object to be serialized. * @return serialized byte array. * @throws IOException occurred while serializing the object to bytes. */ public static<T extends SpecificRecord> byte[] serialize(SpecificDatumWriter<T> datumWriter , T object) throws IOException { ByteArrayOutputStream os = new ByteArrayOutputStream(); serialize(os, datumWriter, object); return os.toByteArray(); }
@Override public void readFields(DataInput in) throws IOException { try { query = IOUtils.deserialize(conf, in, null); } catch (ClassNotFoundException ex) { throw new IOException(ex); } }
/** * Reads the data written by {@link #writeNullFieldsInfo(DataOutput, Object...)} * and returns a boolean array representing whether each field is null or not. * * @param in the input to read from. * @return a boolean[] representing whether each field is null or not. * @throws IOException when value is too long to fit in integer. */ public static boolean[] readNullFieldsInfo(DataInput in) throws IOException { return readBoolArray(in); }
@Override public void write(DataOutput out) throws IOException { super.write(out); IOUtils.serialize(getConf(), out, baseQuery); IOUtils.writeStringArray(out, locations); }
@Override public void readFields(DataInput in) throws IOException { super.readFields(in); try { baseQuery = IOUtils.deserialize(getConf(), in, null); } catch (ClassNotFoundException ex) { throw new IOException(ex); } locations = IOUtils.readStringArray(in); //we should override the data store as basequery's data store //also we may not call super.readFields so that temporary this.dataStore //is not created at all this.dataStore = (DataStoreBase<K, T>) baseQuery.getDataStore(); }
/** * Utility method for deep clone a given AVRO persistent bean instance. * * @param persistent source persistent bean instance. * @param <T> persistent bean type. * @return cloned persistent bean to be returned. */ public static <T extends PersistentBase> T deepClonePersistent(T persistent) { final SpecificDatumWriter<PersistentBase> writer = new SpecificDatumWriter<>(persistent.getSchema()); final byte[] byteData; try { byteData = IOUtils.serialize(writer, persistent); } catch (IOException e) { throw new RuntimeException( "Unable to serialize avro object to byte buffer - " + "please report this issue to the Gora bugtracker " + "or your administrator."); } @SuppressWarnings("unchecked") final SpecificDatumReader<T> reader = new SpecificDatumReader<>((Class<T>) persistent.getClass()); try { return IOUtils.deserialize(byteData, reader, null); } catch (IOException e) { throw new RuntimeException( "Unable to deserialize avro object from byte buffer - " + "please report this issue to the Gora bugtracker " + "or your administrator."); } }
public void write(DataOutput out) throws IOException { //write datastore Text.writeString(out, dataStore.getClass().getCanonicalName()); dataStore.write(out); IOUtils.writeNullFieldsInfo(out, queryString, (fields) , startKey, endKey, filter); if(queryString != null) Text.writeString(out, queryString); if(fields != null) IOUtils.writeStringArray(out, fields); if(startKey != null) IOUtils.serialize(getConf(), out, startKey, dataStore.getKeyClass()); if(endKey != null) IOUtils.serialize(getConf(), out, endKey, dataStore.getKeyClass()); if(filter != null) { Text.writeString(out, filter.getClass().getCanonicalName()); filter.write(out); } WritableUtils.writeVLong(out, getStartTime()); WritableUtils.writeVLong(out, getEndTime()); WritableUtils.writeVLong(out, getLimit()); out.writeBoolean(localFilterEnabled); }
boolean[] nullFields = IOUtils.readNullFieldsInfo(in); fields = IOUtils.readStringArray(in); if(!nullFields[2]) startKey = IOUtils.deserialize(getConf(), in, null, dataStore.getKeyClass()); if(!nullFields[3]) endKey = IOUtils.deserialize(getConf(), in, null, dataStore.getKeyClass()); if(!nullFields[4]) { String filterClass = Text.readString(in);
private void testNullFieldsWith( Object ... values ) throws IOException { DataOutputBuffer out = new DataOutputBuffer(); DataInputBuffer in = new DataInputBuffer(); IOUtils.writeNullFieldsInfo(out, values); in.reset(out.getData(), out.getLength()); boolean[] ret = IOUtils.readNullFieldsInfo(in); //assert assertEquals(values.length, ret.length); for(int i=0; i<values.length; i++) { assertEquals( values[i] == null , ret[i]); } }
/** * Writes a byte[] to the output, representing whether each given field is null * or not. A Vint and ceil( fields.length / 8 ) bytes are written to the output. * * @param out the output to write to. * @param fields the fields to check for null. @see #readNullFieldsInfo(DataInput). * @throws IOException when writing the data to the stream. */ public static void writeNullFieldsInfo(DataOutput out, Object ... fields) throws IOException { boolean[] isNull = new boolean[fields.length]; for(int i=0; i<fields.length; i++) { isNull[i] = (fields[i] == null); } writeBoolArray(out, isNull); }
@Override public void write(DataOutput out) { try{ super.write(out); org.apache.gora.util.IOUtils.writeNullFieldsInfo(out, inputPath, outputPath); if(inputPath != null) Text.writeString(out, inputPath); if(outputPath != null) Text.writeString(out, outputPath); }catch(IOException ex){ LOG.error(ex.getMessage(), ex); } }
@Override public void readFields(DataInput in) throws IOException { try{ super.readFields(in); boolean[] nullFields = org.apache.gora.util.IOUtils.readNullFieldsInfo(in); if(!nullFields[0]) inputPath = Text.readString(in); if(!nullFields[1]) outputPath = Text.readString(in); }catch(IOException ex){ LOG.error(ex.getMessage(), ex); } }
@Override public void readFields(DataInput in) throws IOException { this.arr = IOUtils.readStringArray(in); } @Override
@Override public void write(DataOutput out) throws IOException { IOUtils.writeStringArray(out, arr); } @Override
/** * Utility method for deep clone a given AVRO persistent bean instance. * * @param persistent source persistent bean instance. * @param <T> persistent bean type. * @return cloned persistent bean to be returned. */ public static <T extends PersistentBase> T deepClonePersistent(T persistent) { final SpecificDatumWriter<PersistentBase> writer = new SpecificDatumWriter<>(persistent.getSchema()); final byte[] byteData; try { byteData = IOUtils.serialize(writer, persistent); } catch (IOException e) { throw new RuntimeException( "Unable to serialize avro object to byte buffer - " + "please report this issue to the Gora bugtracker " + "or your administrator."); } @SuppressWarnings("unchecked") final SpecificDatumReader<T> reader = new SpecificDatumReader<>((Class<T>) persistent.getClass()); try { return IOUtils.deserialize(byteData, reader, null); } catch (IOException e) { throw new RuntimeException( "Unable to deserialize avro object from byte buffer - " + "please report this issue to the Gora bugtracker " + "or your administrator."); } }
public void write(DataOutput out) throws IOException { //write datastore Text.writeString(out, dataStore.getClass().getCanonicalName()); dataStore.write(out); IOUtils.writeNullFieldsInfo(out, queryString, (fields) , startKey, endKey, filter); if(queryString != null) Text.writeString(out, queryString); if(fields != null) IOUtils.writeStringArray(out, fields); if(startKey != null) IOUtils.serialize(getConf(), out, startKey, dataStore.getKeyClass()); if(endKey != null) IOUtils.serialize(getConf(), out, endKey, dataStore.getKeyClass()); if(filter != null) { Text.writeString(out, filter.getClass().getCanonicalName()); filter.write(out); } WritableUtils.writeVLong(out, getStartTime()); WritableUtils.writeVLong(out, getEndTime()); WritableUtils.writeVLong(out, getLimit()); out.writeBoolean(localFilterEnabled); }
boolean[] nullFields = IOUtils.readNullFieldsInfo(in); fields = IOUtils.readStringArray(in); if(!nullFields[2]) startKey = IOUtils.deserialize(getConf(), in, null, dataStore.getKeyClass()); if(!nullFields[3]) endKey = IOUtils.deserialize(getConf(), in, null, dataStore.getKeyClass()); if(!nullFields[4]) { String filterClass = Text.readString(in);
@Override public void write(DataOutput out) throws IOException { super.write(out); IOUtils.serialize(getConf(), out, baseQuery); IOUtils.writeStringArray(out, locations); }
@Override public void readFields(DataInput in) throws IOException { super.readFields(in); try { baseQuery = IOUtils.deserialize(getConf(), in, null); } catch (ClassNotFoundException ex) { throw new IOException(ex); } locations = IOUtils.readStringArray(in); //we should override the data store as basequery's data store //also we may not call super.readFields so that temporary this.dataStore //is not created at all this.dataStore = (DataStoreBase<K, T>) baseQuery.getDataStore(); }
/** * Writes a byte[] to the output, representing whether each given field is null * or not. A Vint and ceil( fields.length / 8 ) bytes are written to the output. * * @param out the output to write to. * @param fields the fields to check for null. @see #readNullFieldsInfo(DataInput). * @throws IOException when writing the data to the stream. */ public static void writeNullFieldsInfo(DataOutput out, Object ... fields) throws IOException { boolean[] isNull = new boolean[fields.length]; for(int i=0; i<fields.length; i++) { isNull[i] = (fields[i] == null); } writeBoolArray(out, isNull); }