/** * @return a newly allocated byte buffer of the user defined default size */ private ByteBuf allocateBuffer() { return PooledByteBufAllocator.DEFAULT.buffer(mDataWriters.get(0).chunkSize()); } }
@Override public ByteBuf decode(ByteBuf encoded, int uncompressedLength) throws IOException { ByteBuf uncompressed = PooledByteBufAllocator.DEFAULT.buffer(uncompressedLength, uncompressedLength); ByteBuffer uncompressedNio = uncompressed.nioBuffer(0, uncompressedLength); ByteBuffer encodedNio = encoded.nioBuffer(encoded.readerIndex(), encoded.readableBytes()); decompressor.decompress(encodedNio, encodedNio.position(), uncompressedNio, uncompressedNio.position(), uncompressedNio.remaining()); uncompressed.writerIndex(uncompressedLength); return uncompressed; } }
@Override public ByteBuf encode(ByteBuf source) { int uncompressedLength = source.readableBytes(); int maxLength = compressor.maxCompressedLength(uncompressedLength); ByteBuffer sourceNio = source.nioBuffer(source.readerIndex(), source.readableBytes()); ByteBuf target = PooledByteBufAllocator.DEFAULT.buffer(maxLength, maxLength); ByteBuffer targetNio = target.nioBuffer(0, maxLength); int compressedLength = compressor.compress(sourceNio, 0, uncompressedLength, targetNio, 0, maxLength); target.writerIndex(compressedLength); return target; }
public Buffer allocate(int minimumCapacity) { int size = this.pageSize; while (size < minimumCapacity) { size *= 2; } return new NettyByteBufBuffer(nettyBuffer.buffer(size)); }
private ByteBuf decryptData(SecretKey dataKeySecret, MessageMetadata msgMetadata, ByteBuf payload) { // unpack iv and encrypted data ByteString ivString = msgMetadata.getEncryptionParam(); ivString.copyTo(iv, 0); GCMParameterSpec gcmParams = new GCMParameterSpec(tagLen, iv); ByteBuf targetBuf = null; try { cipher.init(Cipher.DECRYPT_MODE, dataKeySecret, gcmParams); ByteBuffer sourceNioBuf = payload.nioBuffer(payload.readerIndex(), payload.readableBytes()); int maxLength = cipher.getOutputSize(payload.readableBytes()); targetBuf = PooledByteBufAllocator.DEFAULT.buffer(maxLength, maxLength); ByteBuffer targetNioBuf = targetBuf.nioBuffer(0, maxLength); int decryptedSize = cipher.doFinal(sourceNioBuf, targetNioBuf); targetBuf.writerIndex(decryptedSize); } catch (InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException | ShortBufferException e) { log.error("{} Failed to decrypt message {}", logCtx, e.getMessage()); if (targetBuf != null) { targetBuf.release(); targetBuf = null; } } return targetBuf; }
private static ByteBufPair serializeCommandMessageWithSize(BaseCommand cmd, ByteBuf metadataAndPayload) { // / Wire format // [TOTAL_SIZE] [CMD_SIZE][CMD] [MAGIC_NUMBER][CHECKSUM] [METADATA_SIZE][METADATA] [PAYLOAD] // // metadataAndPayload contains from magic-number to the payload included int cmdSize = cmd.getSerializedSize(); int totalSize = 4 + cmdSize + metadataAndPayload.readableBytes(); int headersSize = 4 + 4 + cmdSize; ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize); headers.writeInt(totalSize); // External frame try { // Write cmd headers.writeInt(cmdSize); ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers); cmd.writeTo(outStream); outStream.recycle(); } catch (IOException e) { // This is in-memory serialization, should not fail throw new RuntimeException(e); } return (ByteBufPair) ByteBufPair.get(headers, metadataAndPayload); }
@Override protected DataBuffer getDataBuffer(BlockReadRequestContext context, StreamObserver<ReadResponse> response, long offset, int len) throws Exception { openBlock(context, response); BlockReader blockReader = context.getBlockReader(); Preconditions.checkState(blockReader != null); ByteBuf buf = PooledByteBufAllocator.DEFAULT.buffer(len, len); try { while (buf.writableBytes() > 0 && blockReader.transferTo(buf) != -1) { } return new NettyDataBuffer(buf); } catch (Throwable e) { buf.release(); throw e; } }
void add(MessageImpl<?> msg, SendCallback callback) { if (log.isDebugEnabled()) { log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName, producerName, numMessagesInBatch); } if (++numMessagesInBatch == 1) { // some properties are common amongst the different messages in the batch, hence we just pick it up from // the first message sequenceId = Commands.initBatchMessageMetadata(messageMetadata, msg.getMessageBuilder()); this.firstCallback = callback; batchedMessageMetadataAndPayload = PooledByteBufAllocator.DEFAULT .buffer(Math.min(maxBatchSize, MAX_MESSAGE_BATCH_SIZE_BYTES), PulsarDecoder.MaxMessageSize); } if (previousCallback != null) { previousCallback.addCallback(msg, callback); } previousCallback = callback; currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); PulsarApi.MessageMetadata.Builder msgBuilder = msg.getMessageBuilder(); batchedMessageMetadataAndPayload = Commands.serializeSingleMessageInBatchWithPayload(msgBuilder, msg.getDataBuffer(), batchedMessageMetadataAndPayload); messages.add(msg); msgBuilder.recycle(); }
int totalSize = headerContentSize + payloadSize; ByteBuf metadataAndPayload = PooledByteBufAllocator.DEFAULT.buffer(totalSize, totalSize); try { ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(metadataAndPayload);
@VisibleForTesting public static ByteBuf serializeWithSize(BaseCommand.Builder cmdBuilder) { // / Wire format // [TOTAL_SIZE] [CMD_SIZE][CMD] BaseCommand cmd = cmdBuilder.build(); int cmdSize = cmd.getSerializedSize(); int totalSize = cmdSize + 4; int frameSize = totalSize + 4; ByteBuf buf = PooledByteBufAllocator.DEFAULT.buffer(frameSize, frameSize); // Prepend 2 lengths to the buffer buf.writeInt(totalSize); buf.writeInt(cmdSize); ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(buf); try { cmd.writeTo(outStream); } catch (IOException e) { // This is in-memory serialization, should not fail throw new RuntimeException(e); } finally { cmd.recycle(); cmdBuilder.recycle(); outStream.recycle(); } return buf; }
int checksumReaderIndex = -1; ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize, headersSize);
targetBuf = PooledByteBufAllocator.DEFAULT.buffer(maxLength, maxLength); ByteBuffer targetNioBuf = targetBuf.nioBuffer(0, maxLength);
public ByteBuf getMessageWithMetadata(byte[] data) throws IOException { MessageMetadata messageData = MessageMetadata.newBuilder().setPublishTime(System.currentTimeMillis()) .setProducerName("prod-name").setSequenceId(0).build(); ByteBuf payload = Unpooled.wrappedBuffer(data, 0, data.length); int msgMetadataSize = messageData.getSerializedSize(); int headersSize = 4 + msgMetadataSize; ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize, headersSize); ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers); headers.writeInt(msgMetadataSize); messageData.writeTo(outStream); outStream.recycle(); return ByteBufPair.coalesce(ByteBufPair.get(headers, payload)); }
private ByteBuf getBuffer(Event event) { DatumWriter writer = new FilteredRecordWriter(event.properties().getSchema(), GenericData.get()); ByteBuf buffer = DEFAULT.buffer(100); buffer.writeByte(2); BinaryEncoder encoder = EncoderFactory.get() .directBinaryEncoder(new ByteBufOutputStream(buffer), null); try { encoder.writeString(event.collection()); writer.write(event.properties(), encoder); } catch (Exception e) { throw new RuntimeException("Couldn't serialize event", e); } return buffer; } }
@Override public ByteBuf decode(ByteBuf encoded, int uncompressedLength) throws IOException { ByteBuf uncompressed = PooledByteBufAllocator.DEFAULT.buffer(uncompressedLength, uncompressedLength); ByteBuffer uncompressedNio = uncompressed.nioBuffer(0, uncompressedLength); ByteBuffer encodedNio = encoded.nioBuffer(encoded.readerIndex(), encoded.readableBytes()); decompressor.decompress(encodedNio, encodedNio.position(), uncompressedNio, uncompressedNio.position(), uncompressedNio.remaining()); uncompressed.writerIndex(uncompressedLength); return uncompressed; } }
@Override public ByteBuf decode(ByteBuf encoded, int uncompressedLength) throws IOException { ByteBuf uncompressed = PooledByteBufAllocator.DEFAULT.buffer(uncompressedLength, uncompressedLength); ByteBuffer uncompressedNio = uncompressed.nioBuffer(0, uncompressedLength); ByteBuffer encodedNio = encoded.nioBuffer(encoded.readerIndex(), encoded.readableBytes()); decompressor.decompress(encodedNio, encodedNio.position(), uncompressedNio, uncompressedNio.position(), uncompressedNio.remaining()); uncompressed.writerIndex(uncompressedLength); return uncompressed; } }
@Override public ByteBuf getRoutingKeyData(String key) { int keyLen = key.length(); ByteBuf keyBuf = PooledByteBufAllocator.DEFAULT.buffer(keyLen); keyBuf.writeCharSequence(key, UTF_8); return keyBuf; } }
@Override public ByteBuf encode(ByteBuf source) { int uncompressedLength = source.readableBytes(); int maxLength = compressor.maxCompressedLength(uncompressedLength); ByteBuffer sourceNio = source.nioBuffer(source.readerIndex(), source.readableBytes()); ByteBuf target = PooledByteBufAllocator.DEFAULT.buffer(maxLength, maxLength); ByteBuffer targetNio = target.nioBuffer(0, maxLength); int compressedLength = compressor.compress(sourceNio, 0, uncompressedLength, targetNio, 0, maxLength); target.writerIndex(compressedLength); return target; }
public void fixedNumberEviction(int numEvictions) { ByteBufOutputStream byteStream = new ByteBufOutputStream(PooledByteBufAllocator.DEFAULT.buffer()); try { for (int i = 0; i < numEvictions; i++) { doEvictPage(byteStream); } } finally { byteStream.buffer().release(); } }
public void backgroundEviction() { ByteBufOutputStream byteStream = new ByteBufOutputStream(PooledByteBufAllocator.DEFAULT.buffer()); try { while (shutdownEvictionThreads.get() && shouldEvictPage() && doEvictPage(byteStream)) ; } finally { byteStream.buffer().release(); } }