@Override public void run() { try { InetSocketAddress addr = CrailUtils.getNameNodeAddress(); namenodeServerEp.bind(addr, DaRPCConstants.NAMENODE_DARPC_BACKLOG); LOG.info("opened server at " + addr); while (true) { DaRPCServerEndpoint<DaRPCNameNodeRequest, DaRPCNameNodeResponse> clientEndpoint = namenodeServerEp.accept(); LOG.info("accepting RPC connection, qpnum " + clientEndpoint.getQp().getQp_num()); } } catch(Exception e){ e.printStackTrace(); LOG.error(e.getMessage()); } }
public synchronized void close() throws IOException, InterruptedException { logger.info("shutting down group"); if (closed.get()){ return; } LinkedList<RdmaEndpoint> clientEps = new LinkedList<RdmaEndpoint>(); for (RdmaEndpoint ep : clientEndpointMap.values()) { clientEps.add(ep); } for (RdmaEndpoint ep : clientEps){ ep.close(); } LinkedList<RdmaServerEndpoint<C>> serverEps = new LinkedList<RdmaServerEndpoint<C>>(); for (RdmaServerEndpoint<C> ep : serverEndpointMap.values()) { serverEps.add(ep); } for (RdmaServerEndpoint<C> ep: serverEps){ ep.close(); } cmProcessor.close(); closed.set(true); logger.info("shutting down group done"); }
public synchronized final RdmaServerEndpoint<C> createServerEndpoint() throws IOException{ RdmaCmId idPriv = cmProcessor.createId(RdmaCm.RDMA_PS_TCP); RdmaServerEndpoint<C> ep = new RdmaServerEndpoint<C>(this, idPriv); serverEndpointMap.put(idPriv, ep); return ep; }
private void run() throws Exception { System.out.println("ReadServer, size " + size + ", loop " + loop); RdmaServerEndpoint<ReadServerEndpoint> serverEndpoint = group.createServerEndpoint(); InetAddress ipAddress = InetAddress.getByName(host); InetSocketAddress address = new InetSocketAddress(ipAddress, port); serverEndpoint.bind(address, 10); ReadServer.ReadServerEndpoint endpoint = serverEndpoint.accept(); System.out.println("ReadServer, client connected, address " + address.toString()); //let's send a message to the client //in the message we include the RDMA information of a local buffer which we allow the client to read using a one-sided RDMA operation System.out.println("ReadServer, sending message"); endpoint.sendMessage(); //we have to wait for the CQ event, only then we know the message has been sent out endpoint.takeEvent(); //let's wait for the final message to be received. We don't need to check the message itself, just the CQ event is enough. endpoint.takeEvent(); System.out.println("ReadServer, final message"); //close everything endpoint.close(); serverEndpoint.close(); group.close(); }
public void runRDMA() throws Exception { //we can accept new connections SendRecvServer.CustomServerEndpoint clientEndpoint = serverEndpoint.accept(); //we have previously passed our own endpoint factory to the group, therefore new endpoints will be of type CustomServerEndpoint System.out.println("RDMAvsTcpBenchmarkServer::client connection accepted"); //in our custom endpoints we have prepared (memory registration and work request creation) some memory buffers beforehand. ByteBuffer sendBuf = clientEndpoint.getSendBuf(); sendBuf.asCharBuffer().put("PONG"); ByteBuffer recvBuf = clientEndpoint.getRecvBuf(); SVCPostSend postSend = clientEndpoint.postSend(clientEndpoint.getWrList_send()); SVCPostRecv postRecv = clientEndpoint.postRecv(clientEndpoint.getWrList_recv()); for (int i = 0; i < loopCount + 1; i++){ // Recv PING postRecv.execute(); clientEndpoint.getWcEvents().take(); recvBuf.clear(); //Send PONG postSend.execute(); clientEndpoint.getWcEvents().take(); sendBuf.clear(); } clientEndpoint.close(); serverEndpoint.close(); endpointGroup.close(); }
public void init(CrailConfiguration conf, String[] args) throws Exception { RdmaConstants.init(conf, args); this.serverAddr = StorageUtils.getDataNodeAddress(RdmaConstants.STORAGE_RDMA_INTERFACE, RdmaConstants.STORAGE_RDMA_PORT); if (serverAddr == null){ LOG.info("Configured network interface " + RdmaConstants.STORAGE_RDMA_INTERFACE + " cannot be found..exiting!!!"); return; } this.datanodeGroup = new RdmaPassiveEndpointGroup<RdmaStorageServerEndpoint>(-1, RdmaConstants.STORAGE_RDMA_QUEUESIZE, 4, RdmaConstants.STORAGE_RDMA_QUEUESIZE*100); this.datanodeServerEndpoint = datanodeGroup.createServerEndpoint(); datanodeGroup.init(new RdmaStorageEndpointFactory(datanodeGroup, this)); datanodeServerEndpoint.bind(serverAddr, RdmaConstants.STORAGE_RDMA_BACKLOG); this.allocatedSize = 0; this.fileCount = 0; this.dataDirPath = StorageUtils.getDatanodeDirectory(RdmaConstants.STORAGE_RDMA_DATA_PATH, serverAddr); if (!RdmaConstants.STORAGE_RDMA_PERSISTENT){ StorageUtils.clean(RdmaConstants.STORAGE_RDMA_DATA_PATH, dataDirPath); } }
@Override public void run() { try { this.isAlive = true; LOG.info("rdma storage server started, address " + serverAddr + ", persistent " + RdmaConstants.STORAGE_RDMA_PERSISTENT + ", maxWR " + datanodeGroup.getMaxWR() + ", maxSge " + datanodeGroup.getMaxSge() + ", cqSize " + datanodeGroup.getCqSize()); while(true){ RdmaEndpoint clientEndpoint = datanodeServerEndpoint.accept(); allEndpoints.put(clientEndpoint.getEndpointId(), clientEndpoint); LOG.info("accepting client connection, conncount " + allEndpoints.size()); } } catch(Exception e){ e.printStackTrace(); } this.isAlive = false; }
@Override public StorageResource allocateResource() throws Exception { StorageResource resource = null; if (allocatedSize < RdmaConstants.STORAGE_RDMA_STORAGE_LIMIT){ //mmap buffer int fileId = fileCount++; String dataFilePath = dataDirPath + "/" + fileId; RandomAccessFile dataFile = new RandomAccessFile(dataFilePath, "rw"); if (!RdmaConstants.STORAGE_RDMA_PERSISTENT){ dataFile.setLength(RdmaConstants.STORAGE_RDMA_ALLOCATION_SIZE); } FileChannel dataChannel = dataFile.getChannel(); ByteBuffer dataBuffer = dataChannel.map(MapMode.READ_WRITE, 0, RdmaConstants.STORAGE_RDMA_ALLOCATION_SIZE); dataFile.close(); dataChannel.close(); //register buffer allocatedSize += dataBuffer.capacity(); IbvMr mr = datanodeServerEndpoint.registerMemory(dataBuffer).execute().free().getMr(); //create resource resource = StorageResource.createResource(mr.getAddr(), mr.getLength(), mr.getLkey()); } return resource; }
serverEndpointMap.get(idPriv).dispatchCmEvent(cmEvent); serverEndpointMap.get(idPriv).dispatchCmEvent(cmEvent); serverEndpointMap.get(idPriv).dispatchCmEvent(cmEvent);
public synchronized void dispatchCmEvent(RdmaCmEvent cmEvent) throws IOException { try { int eventType = cmEvent.getEvent(); if (eventType == RdmaCmEvent.EventType.RDMA_CM_EVENT_CONNECT_REQUEST.ordinal()) { // logger.info("got event type + RDMA_CM_EVENT_CONNECT_REQUEST, serverAddress " + this.getSrcAddr()); RdmaCmId connId = cmEvent.getConnIdPriv(); C ep = group.createEndpoint(connId); ep.dispatchCmEvent(cmEvent); requested.add(ep); notifyAll(); } else if (eventType == RdmaCmEvent.EventType.RDMA_CM_EVENT_DISCONNECTED .ordinal()) { // logger.info("got event type + RDMA_CM_EVENT_CONNECT_REQUEST, serverAddress " + this.getSrcAddr()); connState = CONN_STATE_CLOSED; notifyAll(); } else { logger.info("got event type + UNKNOWN, serverAddress " + this.getSrcAddr()); } } catch(Exception e){ throw new IOException(e); } }
InetAddress ipAddress = InetAddress.getByName(host); InetSocketAddress address = new InetSocketAddress(ipAddress, port); serverEndpoint.bind(address, 10); SendRecvServer.SendRecvEndpoint endpoint = serverEndpoint.accept(); System.out.println("SendRecvServer, client connected, address " + address.toString()); serverEndpoint.close(); group.close();
serverEndpoint.bind(rdmaAddress, 10); System.out.println("RdmaVsTcpBenchmarkServer bound to address " + rdmaAddress.toString());
synchronized IbvPd createProtectionDomainRaw(RdmaServerEndpoint<C> endpoint) throws IOException{ return createProtectionDomain(serverEndpointMap.get(endpoint.getIdPriv())); }
public synchronized IbvPd createProtectionDomain(RdmaServerEndpoint<?> endpoint) throws IOException { IbvContext context = endpoint.getIdPriv().getVerbs(); return createProtectionDomain(context); }