public IbvWC clone(){ IbvWC wc = new IbvWC(); wc.byte_len = this.byte_len; wc.diff = this.diff; wc.dlid_path_bits = this.dlid_path_bits; wc.err = this.err; wc.imm_data = this.imm_data; wc.isSend = this.isSend; wc.opcode = this.opcode; wc.pkey_index = this.pkey_index; wc.qp_num = this.qp_num; wc.sl = this.sl; wc.slid = this.slid; wc.wc_flags = this.wc_flags; wc.status = this.status; wc.wqIndex = this.wqIndex; wc.wr_id = this.wr_id; return wc; }
private void update(IbvWC wc, ByteBuffer buffer) { int initialPos = buffer.position(); wc.setWr_id(buffer.getLong()); wc.setStatus(buffer.getInt()); wc.setOpcode(buffer.getInt()); if (wc.getStatus() == IbvWC.IbvWcStatus.IBV_WC_SUCCESS.ordinal()){ wc.setVendor_err(buffer.getInt()); wc.setByte_len(buffer.getInt()); wc.setImm_data(buffer.getInt()); wc.setQp_num(buffer.getInt()); wc.setSrc_qp(buffer.getInt()); wc.setWc_flags(buffer.getInt()); wc.setPkey_index(buffer.getShort()); wc.setSlid(buffer.getShort()); wc.setSl(buffer.get()); wc.setDlid_path_bits(buffer.get()); } int newPosition = initialPos + IbvWC.CSIZE; buffer.position(newPosition); }
@Override public void dispatchCqEvent(IbvWC wc) throws IOException { if (wc.getStatus() == 0){ RdmaActiveFuture future = futureMap.remove(wc.getWr_id()); if (future != null){ future.signal(); if (future.isWrite()){ sendQueueAvailable.release(2); } else { sendQueueAvailable.release(); } } else { throw new IOException("cannot find future object for wrid " + wc.getWr_id() + ", status " + wc.getStatus() + ", opcount " + opcount + ", wc.qpnum " + wc.getQp_num() + ", this.qp.num " + this.qp.getQp_num() + ", connstate " + this.getConnState() + ", futureMap.size " + futureMap.size()); } } else if (wc.getStatus() == 5){ } else { throw new IOException("error in wc, status " + wc.getStatus()); } }
public void dispatchCqEvent(IbvWC wc) throws IOException { if (wc.getStatus() == 5){ //flush return; } else if (wc.getStatus() != 0){ throw new IOException("Faulty operation! wc.status " + wc.getStatus()); } if (wc.getOpcode() == 128){ //receiving a message int index = (int) wc.getWr_id(); ByteBuffer recvBuffer = recvBufs[index]; int ticket = recvBuffer.getInt(0); recvBuffer.position(4); dispatchReceive(recvBuffer, ticket, index); } else if (wc.getOpcode() == 0) { //send completion int index = (int) wc.getWr_id(); ByteBuffer sendBuffer = sendBufs[index]; int ticket = sendBuffer.getInt(0); dispatchSend(ticket); } else { throw new IOException("Unkown opcode " + wc.getOpcode()); } }
private void awaitSends() throws IOException { while (sendPending > 0) { int res = poll.execute().getPolls(); if (res > 0){ for (int i = 0; i < res; i++){ if (wcList[i].getOpcode() == 128){ // System.out.println("recv from wrid " + wcList[i].getWr_id()); recvBudget++; recvPending--; } else { sendBudget++; sendPending--; } } } } }
break; } else if (res > 0){ if (wcList[0].getStatus() == IbvWC.IbvWcStatus.IBV_WC_SUCCESS.ordinal()){ } else { break;
System.out.println("SimpleClient::message sent, wr_id " + wc.getWr_id());
public abstract void dispatchCqEvent(C endpoint, IbvWC wc) throws IOException;
private void awaitRecvs() throws IOException { while (recvPending > 0) { int res = poll.execute().getPolls(); if (res > 0){ for (int i = 0; i < res; i++){ if (wcList[i].getOpcode() == 128){ // System.out.println("recv from wrid " + wcList[i].getWr_id()); recvBudget++; recvPending--; } else { sendBudget++; sendPending--; } } } } }
private void dispatchCqEvent(IbvWC wc) throws IOException { if (wc.getStatus() == 5){ // logger.info("flush wc"); } else if (wc.getStatus() != 0){ LOG.info("faulty request, status " + wc.getStatus()); } else { RdmaPassiveFuture future = futureMap.remove(wc.getWr_id()); if (future != null){ future.signal(wc.getStatus()); if (future.isWrite()){ sendQueueAvailable.release(2); } else { sendQueueAvailable.release(); } } else { throw new IOException("cannot find future object for wrid " + wc.getWr_id() + ", status " + wc.getStatus() + ", opcount " + opcount + ", ep " + this.getEndpointId() + ", wc.qpnum " + wc.getQp_num() + ", this.qp.num " + this.qp.getQp_num() + ", connstate " + this.getConnState() + ", futureMap.size " + futureMap.size()); } } }
public RdmaCqProcessor(IbvContext context, int cqSize, int wrSize, long affinity, int clusterId, int timeout, boolean polling) throws IOException { super(context, cqSize); this.clusterId = clusterId; this.affinity = affinity; this.running = false; this.wrSize = Math.min(cqSize, wrSize); this.wcList = new IbvWC[this.wrSize]; for (int i = 0; i < wcList.length; i++){ wcList[i] = new IbvWC(); } this.blocking = !polling; this.ackCounter = 0; this.reqNotify = cq.reqNotification(false); this.poll = cq.poll(wcList, wcList.length); this.timeout = timeout; if (blocking){ reqNotify.execute(); } this.qpMap = new ConcurrentHashMap<Integer, C>(); this.thread = new Thread(this); }
private void awaitRecvs() throws IOException { while (recvPending > 0) { int res = poll.execute().getPolls(); if (res > 0){ for (int i = 0; i < res; i++){ if (wcList[i].getOpcode() == 128){ // System.out.println("recv from wrid " + wcList[i].getWr_id()); recvBudget++; recvPending--; } else { sendBudget++; sendPending--; } } } } }
private SVCPollCq getPollCqCall(int size) throws Exception{ if (!CachingON || pollCqCall == null || !pollCqCall.isValid()) { wcList = new IbvWC[size]; for (int i = 0; i < size; i++){ wcList[i] = new IbvWC(); } pollCqCall = cq.poll(wcList, size); } return pollCqCall; } }
private void awaitSends() throws IOException { while (sendPending > 0) { int res = poll.execute().getPolls(); if (res > 0){ for (int i = 0; i < res; i++){ if (wcList[i].getOpcode() == 128){ // System.out.println("recv from wrid " + wcList[i].getWr_id()); recvBudget++; recvPending--; } else { sendBudget++; sendPending--; } } } } }
@Override protected synchronized void init() throws IOException { super.init(); for (int i = 0; i < RdmaConstants.STORAGE_RDMA_QUEUESIZE; i++){ SVCPostSend write = initWriteOp(); writeOps.add(write); SVCPostSend read = initReadOp(); readOps.add(read); } IbvCQ cq = getCqProvider().getCQ(); this.wcList = new IbvWC[getCqProvider().getCqSize()]; for (int i = 0; i < wcList.length; i++){ wcList[i] = new IbvWC(); } this.poll = cq.poll(wcList, wcList.length); }
private int pollRecvs() throws IOException { if (recvPending == 0){ return 0; } int recvCount = 0; while (recvCount == 0){ int res = 0; while (res == 0) { res = poll.execute().getPolls(); } for (int i = 0; i < res; i++){ if (wcList[i].getOpcode() == 128){ // System.out.println("recv from wrid " + wcList[i].getWr_id()); recvCount++; recvBudget++; recvPending--; } else { sendBudget++; sendPending--; } } } return recvCount; }
@Override public void init() throws IOException { super.init(); IbvCQ cq = getCqProvider().getCQ(); this.wcList = new IbvWC[getCqProvider().getCqSize()]; for (int i = 0; i < wcList.length; i++){ wcList[i] = new IbvWC(); } this.poll = cq.poll(wcList, wcList.length); }
private int pollRecvs() throws IOException { if (recvPending == 0){ return 0; } int recvCount = 0; while (recvCount == 0){ int res = 0; while (res == 0) { res = poll.execute().getPolls(); } for (int i = 0; i < res; i++){ if (wcList[i].getOpcode() == 128){ // System.out.println("recv from wrid " + wcList[i].getWr_id()); recvCount++; recvBudget++; recvPending--; } else { sendBudget++; sendPending--; } } } return recvCount; }
@Override protected synchronized void init() throws IOException { super.init(); IbvCQ cq = getCqProvider().getCQ(); this.wcList = new IbvWC[getCqProvider().getCqSize()]; for (int i = 0; i < wcList.length; i++){ wcList[i] = new IbvWC(); } this.poll = cq.poll(wcList, wcList.length); for(int i = 0; i < pipelineLength; i++){ recvBufs[i] = ByteBuffer.allocateDirect(bufferSize); sendBufs[i] = ByteBuffer.allocateDirect(bufferSize); this.recvCall[i] = setupRecvTask(recvBufs[i], i); this.sendCall[i] = setupSendTask(sendBufs[i], i); } }
@Override protected synchronized void init() throws IOException { super.init(); IbvCQ cq = getCqProvider().getCQ(); this.wcList = new IbvWC[getCqProvider().getCqSize()]; for (int i = 0; i < wcList.length; i++){ wcList[i] = new IbvWC(); } this.poll = cq.poll(wcList, wcList.length); for(int i = 0; i < pipelineLength; i++){ recvBufs[i] = ByteBuffer.allocateDirect(bufferSize); sendBufs[i] = ByteBuffer.allocateDirect(bufferSize); this.recvCall[i] = setupRecvTask(recvBufs[i], i); this.sendCall[i] = setupSendTask(sendBufs[i], i); } this.installRecvs(pipelineLength); }