@Override public ChannelBuffer copy(int index, int length) { ByteBuffer src; try { src = (ByteBuffer) buffer.duplicate().position(index).limit(index + length); } catch (IllegalArgumentException e) { throw new IndexOutOfBoundsException(); } ByteBuffer dst = buffer.isDirect() ? ByteBuffer.allocateDirect(length) : ByteBuffer.allocate(length); dst.put(src); dst.clear(); return new ByteBufferBackedChannelBuffer(dst); }
public void setBuffer(@Nonnull ByteBuffer buffer) { if (buffer.hasArray()) { this.buffer = buffer.array(); this.position = buffer.arrayOffset() + buffer.position(); this.end = this.position + buffer.remaining(); } else if (buffer.isDirect() || buffer.isReadOnly()) { // TODO: FLINK-8585 handle readonly and other non array based buffers more efficiently without data copy this.buffer = new byte[buffer.remaining()]; this.position = 0; this.end = this.buffer.length; buffer.get(this.buffer); } else { throw new IllegalArgumentException("The given buffer is neither an array-backed heap ByteBuffer, nor a direct ByteBuffer."); } }
/** * Compress the content in the given input buffer. After the compression, * you can retrieve the compressed data from the output buffer [pos() ... * limit()) (compressed data size = limit() - pos() = remaining()) * * @param uncompressed buffer[pos() ... limit()) containing the input data * @param compressed output of the compressed data. Uses range [pos()..]. * @return byte size of the compressed data. * @throws SnappyError when the input is not a direct buffer */ public static int compress(ByteBuffer uncompressed, ByteBuffer compressed) throws IOException { if (!uncompressed.isDirect()) { throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer"); } if (!compressed.isDirect()) { throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer"); } // input: uncompressed[pos(), limit()) // output: compressed int uPos = uncompressed.position(); int uLen = uncompressed.remaining(); int compressedSize = impl.rawCompress(uncompressed, uPos, uLen, compressed, compressed.position()); // pos limit // [ ......BBBBBBB.........] compressed.limit(compressed.position() + compressedSize); return compressedSize; }
@Override public ChannelBuffer getBuffer(ByteBuffer nioBuffer) { if (!nioBuffer.isReadOnly() && nioBuffer.isDirect()) { return ChannelBuffers.wrappedBuffer(nioBuffer); } ChannelBuffer buf = getBuffer(nioBuffer.remaining()); int pos = nioBuffer.position(); buf.writeBytes(nioBuffer); nioBuffer.position(pos); return buf; }
/** * For direct buffers, calls {@link Pointer#Pointer(Buffer)}, while for buffers * backed with an array, allocates enough memory for the array and copies it. * * @param buffer the Buffer to reference or copy * @see #put(byte[]) */ public BytePointer(ByteBuffer buffer) { super(buffer); if (buffer != null && !buffer.isDirect() && buffer.hasArray()) { byte[] array = buffer.array(); allocateArray(array.length - buffer.arrayOffset()); put(array, buffer.arrayOffset(), array.length - buffer.arrayOffset()); position(buffer.position()); limit(buffer.limit()); } } /**
/** * @param src Source. * @param srcPos Source pos. * @param dest Destination. * @param destPos Destination pos. * @param len Length. */ private void copy(ByteBuffer src, int srcPos, ByteBuffer dest, int destPos, int len) { assert mode != MAPPED; if (buf.isDirect()) { ByteBuffer src0 = src.duplicate(); src0.limit(srcPos + len); src0.position(srcPos); ByteBuffer dest0 = dest.duplicate(); dest0.limit(destPos + len); dest0.position(destPos); dest0.put(src0); } else System.arraycopy(src.array(), srcPos, buf.array(), destPos, len); }
/** * Reads bytes at the given offset as a short value. * @param buf * @param offset * @return short value at offset */ static short getAsShort(ByteBuffer buf, int offset) { if (buf.isDirect()) { return theUnsafe.getShort(((DirectBuffer) buf).address() + offset); } return theUnsafe.getShort(buf.array(), BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset); }
/** * @param src Source. * @param srcOff Source offset in bytes. * @param dst Destination. * @param dstOff Destination offset in bytes. * @param cnt Bytes count to copy. */ public static void copyMemory(ByteBuffer src, long srcOff, ByteBuffer dst, long dstOff, long cnt) { byte[] srcArr = src.hasArray() ? src.array() : null; byte[] dstArr = dst.hasArray() ? dst.array() : null; long srcArrOff = src.hasArray() ? src.arrayOffset() + GridUnsafe.BYTE_ARR_OFF : 0; long dstArrOff = dst.hasArray() ? dst.arrayOffset() + GridUnsafe.BYTE_ARR_OFF : 0; long srcPtr = src.isDirect() ? GridUnsafe.bufferAddress(src) : 0; long dstPtr = dst.isDirect() ? GridUnsafe.bufferAddress(dst) : 0; GridUnsafe.copyMemory(srcArr, srcPtr + srcArrOff + srcOff, dstArr, dstPtr + dstArrOff + dstOff, cnt); }
@Test public void testChunkCreateDirectByteBuffer() { SlabImpl slab = new SlabImpl(1024 * 1024); try { MemoryAllocatorImpl ma = MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(), new NullOffHeapMemoryStats(), new SlabImpl[] {slab}); ByteBuffer bb = ByteBuffer.allocate(1024); for (int i = 0; i < 1024; i++) { bb.put((byte) i); } bb.position(0); OffHeapStoredObject c = (OffHeapStoredObject) ma.allocateAndInitialize(bb.array(), false, false); assertEquals(1024, c.getDataSize()); if (!Arrays.equals(bb.array(), c.getRawBytes())) { fail("arrays are not equal. Expected " + Arrays.toString(bb.array()) + " but found: " + Arrays.toString(c.getRawBytes())); } ByteBuffer dbb = c.createDirectByteBuffer(); assertEquals(true, dbb.isDirect()); assertEquals(bb, dbb); } finally { MemoryAllocatorImpl.freeOffHeapMemory(); } }
if (targetBuffer.remaining() == 0) { return; if (targetBuffer.isDirect()) { ByteBuffer directCheckSumBuf = BUFFER_POOL.getBuffer(true, stripedWriter.getChecksumBuf().length); } else { stripedWriter.getChecksum().calculateChunkedSums( targetBuffer.array(), 0, targetBuffer.remaining(), stripedWriter.getChecksumBuf(), 0); while (targetBuffer.remaining() > 0) { DFSPacket packet = new DFSPacket(packetBuf, stripedWriter.getMaxChunksPerPacket(),
/** * Wrap the visible portion of a {@link java.nio.ByteBuffer}. */ public static Slice wrappedBuffer(ByteBuffer buffer) { if (buffer.isDirect()) { long address = getAddress(buffer); return new Slice(null, address + buffer.position(), buffer.remaining(), buffer.capacity(), buffer); } if (buffer.hasArray()) { return new Slice(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); } throw new IllegalArgumentException("cannot wrap " + buffer.getClass().getName()); }
private static void decompressChunk( ByteBuffer src, CompressionCodec codec, ByteBuffer dest) throws IOException { int startPos = dest.position(), startLim = dest.limit(); int startSrcPos = src.position(), startSrcLim = src.limit(); if (LOG.isTraceEnabled()) { LOG.trace("Decompressing " + src.remaining() + " bytes to dest buffer pos " + dest.position() + ", limit " + dest.limit()); } codec.reset(); // We always need to call reset on the codec. codec.decompress(src, dest); dest.position(startPos); int newLim = dest.limit(); if (newLim > startLim) { throw new AssertionError("After codec, buffer [" + startPos + ", " + startLim + ") became [" + dest.position() + ", " + newLim + ")"); } if (dest.remaining() == 0) { throw new IOException("The codec has produced 0 bytes for {" + src.isDirect() + ", " + src.position() + ", " + src.remaining() + "} into {" + dest.isDirect() + ", " + dest.position() + ", " + dest.remaining() + "}"); } }
protected void checkSizeAndGrow(int extra) { long capacityNeeded = curBuf.position() + (long) extra; if (capacityNeeded > curBuf.limit()) { // guarantee it's possible to fit if (capacityNeeded > MAX_ARRAY_SIZE) { throw new BufferOverflowException(); } // double until hit the cap long nextCapacity = Math.min(curBuf.capacity() * 2L, MAX_ARRAY_SIZE); // but make sure there is enough if twice the existing capacity is still too small nextCapacity = Math.max(nextCapacity, capacityNeeded); ByteBuffer newBuf = allocate((int) nextCapacity, curBuf.isDirect()); curBuf.flip(); ByteBufferUtils.copyFromBufferToBuffer(curBuf, newBuf); curBuf = newBuf; } }
@Override public ChannelBuffer getBuffer(ByteBuffer nioBuffer) { if (!nioBuffer.isReadOnly() && nioBuffer.isDirect()) { return ChannelBuffers.wrappedBuffer(nioBuffer); } ChannelBuffer buf = getBuffer(nioBuffer.remaining()); int pos = nioBuffer.position(); buf.writeBytes(nioBuffer); nioBuffer.position(pos); return buf; }
/** * For direct buffers, calls {@link Pointer#Pointer(Buffer)}, while for buffers * backed with an array, allocates enough memory for the array and copies it. * * @param buffer the Buffer to reference or copy * @see #put(boolean[]) */ public BooleanPointer(ByteBuffer buffer) { super(buffer); if (buffer != null && !buffer.isDirect() && buffer.hasArray()) { byte[] array = buffer.array(); allocateArray(array.length - buffer.arrayOffset()); for (int i = buffer.arrayOffset(); i < array.length; i++) { put(i - buffer.arrayOffset(), array[i] != 0); } position(buffer.position()); limit(buffer.limit()); } } /**
/** * Reads bytes at the given offset as an int value. * @param buf * @param offset * @return int value at offset */ static int getAsInt(ByteBuffer buf, int offset) { if (buf.isDirect()) { return theUnsafe.getInt(((DirectBuffer) buf).address() + offset); } return theUnsafe.getInt(buf.array(), BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset); }
/** * Decodes the given portion of the {@link ByteBuffer} into a {@link String}. * * @throws InvalidProtocolBufferException if the portion of the buffer is not valid UTF-8. */ final String decodeUtf8(ByteBuffer buffer, int index, int size) throws InvalidProtocolBufferException { if (buffer.hasArray()) { final int offset = buffer.arrayOffset(); return decodeUtf8(buffer.array(), offset + index, size); } else if (buffer.isDirect()) { return decodeUtf8Direct(buffer, index, size); } return decodeUtf8Default(buffer, index, size); }