FanOutOneBlockAsyncDFSOutput(Configuration conf, FSUtils fsUtils, DistributedFileSystem dfs, DFSClient client, ClientProtocol namenode, String clientName, String src, long fileId, LocatedBlock locatedBlock, Encryptor encryptor, List<Channel> datanodeList, DataChecksum summer, ByteBufAllocator alloc) { this.conf = conf; this.fsUtils = fsUtils; this.dfs = dfs; this.client = client; this.namenode = namenode; this.fileId = fileId; this.clientName = clientName; this.src = src; this.block = locatedBlock.getBlock(); this.locations = locatedBlock.getLocations(); this.encryptor = encryptor; this.datanodeList = datanodeList; this.summer = summer; this.maxDataLen = MAX_DATA_LEN - (MAX_DATA_LEN % summer.getBytesPerChecksum()); this.alloc = alloc; this.buf = alloc.directBuffer(sendBufSizePRedictor.initialSize()); this.state = State.STREAMING; setupReceiver(conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, READ_TIMEOUT)); }
@Override protected ByteBuf allocateBuffer(ChannelHandlerContext ctx, ByteBuf msg, boolean preferDirect) throws Exception { if (preferDirect) { return ctx.alloc().directBuffer(msg.readableBytes()); } else { return ctx.alloc().buffer(msg.readableBytes()); } }
private void endBlock() throws IOException { Preconditions.checkState(waitingAckQueue.isEmpty(), "should call flush first before calling close"); if (state != State.STREAMING) { throw new IOException("stream already broken"); } state = State.CLOSING; long finalizedLength = ackedBlockLength; PacketHeader header = new PacketHeader(4, finalizedLength, nextPacketSeqno, true, 0, false); buf.release(); buf = null; int headerLen = header.getSerializedSize(); ByteBuf headerBuf = alloc.directBuffer(headerLen); header.putInBuffer(headerBuf.nioBuffer(0, headerLen)); headerBuf.writerIndex(headerLen); CompletableFuture<Long> future = new CompletableFuture<>(); waitingAckQueue.add(new Callback(future, finalizedLength, datanodeList)); datanodeList.forEach(ch -> ch.writeAndFlush(headerBuf.retainedDuplicate())); headerBuf.release(); try { future.get(); } catch (InterruptedException e) { throw (IOException) new InterruptedIOException().initCause(e); } catch (ExecutionException e) { Throwable cause = e.getCause(); Throwables.propagateIfPossible(cause, IOException.class); throw new IOException(cause); } }
@Override protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) throws Exception { ByteBuf inBuf; boolean release = false; if (msg.nioBufferCount() == 1) { inBuf = msg; } else { inBuf = ctx.alloc().directBuffer(msg.readableBytes()); msg.readBytes(inBuf); release = true; } ByteBuffer inBuffer = inBuf.nioBuffer(); ByteBuf outBuf = ctx.alloc().directBuffer(inBuf.readableBytes()); ByteBuffer outBuffer = outBuf.nioBuffer(0, inBuf.readableBytes()); decryptor.decrypt(inBuffer, outBuffer); outBuf.writerIndex(inBuf.readableBytes()); if (release) { inBuf.release(); } ctx.fireChannelRead(outBuf); } }
ByteBuf encryptBuf = alloc.directBuffer(dataLen); buf.readBytes(encryptBuf, trailingPartialChunkLength); int toEncryptLength = dataLen - trailingPartialChunkLength; ByteBuf newBuf = alloc.directBuffer(sendBufSizePRedictor.guess(dataLen)) .ensureWritable(trailingPartialChunkLength); if (trailingPartialChunkLength != 0) {
@Override protected void encode(ChannelHandlerContext ctx, ByteBuf msg, ByteBuf out) throws Exception { ByteBuf inBuf; boolean release = false; if (msg.nioBufferCount() == 1) { inBuf = msg; } else { inBuf = ctx.alloc().directBuffer(msg.readableBytes()); msg.readBytes(inBuf); release = true; } ByteBuffer inBuffer = inBuf.nioBuffer(); ByteBuffer outBuffer = out.nioBuffer(0, inBuf.readableBytes()); encryptor.encrypt(inBuffer, outBuffer); out.writerIndex(inBuf.readableBytes()); if (release) { inBuf.release(); } } }
int numChecks = dataLen / chunkLen + (trailingPartialChunkLen != 0 ? 1 : 0); int checksumLen = numChecks * summer.getChecksumSize(); ByteBuf checksumBuf = alloc.directBuffer(checksumLen); summer.calculateChunkedSums(dataBuf.nioBuffer(), checksumBuf.nioBuffer(0, checksumLen)); checksumBuf.writerIndex(checksumLen);
/** * Creates a new big-endian direct buffer with reasonably small initial capacity, which * expands its capacity boundlessly on demand. */ public static ByteBuf directBuffer() { return ALLOC.directBuffer(); }
/** * Creates a new big-endian direct buffer with the specified {@code capacity}, which * expands its capacity boundlessly on demand. The new buffer's {@code readerIndex} and * {@code writerIndex} are {@code 0}. */ public static ByteBuf directBuffer(int initialCapacity) { return ALLOC.directBuffer(initialCapacity); }
/** * Creates a new big-endian direct buffer with the specified * {@code initialCapacity}, that may grow up to {@code maxCapacity}. * The new buffer's {@code readerIndex} and {@code writerIndex} are * {@code 0}. */ public static ByteBuf directBuffer(int initialCapacity, int maxCapacity) { return ALLOC.directBuffer(initialCapacity, maxCapacity); }
@Override public ByteBuf directBuffer() { return allocator.directBuffer(); }
@Override public ByteBuf directBuffer(int initialCapacity, int maxCapacity) { return allocator.directBuffer(initialCapacity, maxCapacity); }
@Override public ByteBuf directBuffer(int initialCapacity) { return allocator.directBuffer(initialCapacity); }
@Override public AllocatedBuffer allocateDirectBuffer(int capacity) { return new BufferAdapter(alloc.directBuffer(capacity)); } }
@Override public ByteBuf copy(int index, int length) { ensureAccessible(); ByteBuffer src; try { src = (ByteBuffer) buffer.duplicate().clear().position(index).limit(index + length); } catch (IllegalArgumentException ignored) { throw new IndexOutOfBoundsException("Too many bytes to read - Need " + (index + length)); } return alloc().directBuffer(length, maxCapacity()).writeBytes(src); }
/** * Always prefer a direct buffer when it's pooled, so that we reduce the number of memory copies * in {@link OpenSslEngine}. */ private ByteBuf allocate(ChannelHandlerContext ctx, int capacity) { ByteBufAllocator alloc = ctx.alloc(); if (engineType.wantsDirectBuffer) { return alloc.directBuffer(capacity); } else { return alloc.buffer(capacity); } }
private static ByteBuf newDirectBuffer0(Object holder, ByteBuf buf, ByteBufAllocator alloc, int capacity) { final ByteBuf directBuf = alloc.directBuffer(capacity); directBuf.writeBytes(buf, buf.readerIndex(), capacity); ReferenceCountUtil.safeRelease(holder); return directBuf; }
private static ByteBuf newDirectBuffer0(Object holder, ByteBuf buf, ByteBufAllocator alloc, int capacity) { final ByteBuf directBuf = alloc.directBuffer(capacity); directBuf.writeBytes(buf, buf.readerIndex(), capacity); ReferenceCountUtil.safeRelease(holder); return directBuf; }
@Override public ByteBuf copy(int index, int length) { checkIndex(index, length); ByteBuf copy = alloc().directBuffer(length, maxCapacity()); copy.writeBytes(this, index, length); return copy; }
static ByteBuf copy(AbstractByteBuf buf, long addr, int index, int length) { buf.checkIndex(index, length); ByteBuf copy = buf.alloc().directBuffer(length, buf.maxCapacity()); if (length != 0) { if (copy.hasMemoryAddress()) { PlatformDependent.copyMemory(addr, copy.memoryAddress(), length); copy.setIndex(0, length); } else { copy.writeBytes(buf, index, length); } } return copy; }