/** * * @param messageValueFactory message value factory * @param pointers pointers to messages in buffer * @param msgBuffer holds the byte arrays of serialized messages */ public PointerListMessagesIterable(MessageValueFactory<M> messageValueFactory, LongArrayList pointers, ExtendedByteArrayOutputBuffer msgBuffer) { this.messageValueFactory = messageValueFactory; this.pointers = pointers; this.msgBuffer = msgBuffer; // TODO - if needed implement same for Safe as well messageReader = new UnsafeReusableByteArrayInput(); }
/** * Deserialize from given byte array into given writable, * using given reusable UnsafeReusableByteArrayInput. * * @param data Byte array representing writable * @param to Object to fill * @param reusableInput Reusable input to use * @param <T> Type of the object */ public static <T extends Writable> void fromByteArrayUnsafe( byte[] data, T to, UnsafeReusableByteArrayInput reusableInput) { try { reusableInput.initialize(data, 0, data.length); to.readFields(reusableInput); if (reusableInput.available() != 0) { throw new RuntimeException( "Serialization encountered issues, " + reusableInput.available() + " bytes left to be read"); } } catch (IOException e) { throw new RuntimeException(e); } }
@Override public M next() { long pointer = iterator.nextLong(); try { int index = (int) (pointer >>> 32); int offset = (int) pointer; ExtendedDataOutput buffer = msgBuffer.getDataOutput(index); messageReader.initialize(buffer.getByteArray(), offset, buffer.getPos()); reusableMsg.readFields(messageReader); } catch (IOException e) { throw new IllegalStateException("Got exception : " + e); } return reusableMsg; }
private <W extends Writable> W copyWritable(W writable) { if (reusedOut == null) { reusedOut = new UnsafeByteArrayOutputStream(); } if (reusedIn == null) { reusedIn = new UnsafeReusableByteArrayInput(); } return WritableUtils.createCopy(reusedOut, reusedIn, writable, null); }
reusableOut.reset(); original.write(reusableOut); reusableIn.initialize( reusableOut.getByteArray(), 0, reusableOut.getPos()); copy.readFields(reusableIn); if (reusableIn.available() != 0) { throw new RuntimeException("Serialization of " + original.getClass() + " encountered issues, " + reusableIn.available() + " bytes left to be read");
/** * Constructor * * Creates new instances of all reducers from * {@link WorkerAggregatorHandler} */ public ThreadLocalWorkerGlobalCommUsage() { threadReducerMap = Maps.newHashMapWithExpectedSize( WorkerAggregatorHandler.this.reducerMap.size()); UnsafeByteArrayOutputStream out = new UnsafeByteArrayOutputStream(); UnsafeReusableByteArrayInput in = new UnsafeReusableByteArrayInput(); for (Entry<String, Reducer<Object, Writable>> entry : reducerMap.entrySet()) { ReduceOperation<Object, Writable> globalReduceOp = entry.getValue().getReduceOp(); ReduceOperation<Object, Writable> threadLocalCopy = WritableUtils.createCopy(out, in, globalReduceOp, conf); threadReducerMap.put(entry.getKey(), new Reducer<>(threadLocalCopy)); } }
new KryoWritableWrapper<>(); WritableUtils.fromByteArrayUnsafe( merged, wrapper, new UnsafeReusableByteArrayInput()); return wrapper.get();
/** * Get object from a conf option using kryo * * @param confOption Conf option * @param conf Configuration * @return Object from conf * @param <T> Type of the object */ public static <T> T getObjectKryo(String confOption, Configuration conf) { KryoWritableWrapper<T> wrapper = new KryoWritableWrapper<>(); WritableUtils.fromByteArrayUnsafe( getByteArray(confOption, conf), wrapper, new UnsafeReusableByteArrayInput()); return wrapper.get(); } }
@Override public void doRequest(ServerData serverData) { UnsafeByteArrayOutputStream reusedOut = new UnsafeByteArrayOutputStream(); UnsafeReusableByteArrayInput reusedIn = new UnsafeReusableByteArrayInput();