From 0fe313b000b6a699afbbb59ef9c47a2b22146f1e Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Thu, 30 Nov 2006 16:37:15 -0800 Subject: [PATCH] RDMA/cma: Allow early transition to RTS to handle lost CM messages During connection establishment, the passive side of a connection can receive messages from the active side before the connection event has been delivered to the user. Allow the passive side to send messages in response to received data before the event is delivered. To handle the case where the connection messages are lost, a new rdma_notify() function is added that users may invoke to force a connection into the established state. Signed-off-by: Sean Hefty Signed-off-by: Roland Dreier --- drivers/infiniband/core/cm.c | 4 +++ drivers/infiniband/core/cma.c | 68 ++++++++++++++++++++++------------- include/rdma/rdma_cm.h | 15 ++++++++ 3 files changed, 63 insertions(+), 24 deletions(-) diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 79c937bf69..d446998b12 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -3289,6 +3289,10 @@ static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv, spin_lock_irqsave(&cm_id_priv->lock, flags); switch (cm_id_priv->id.state) { + /* Allow transition to RTS before sending REP */ + case IB_CM_REQ_RCVD: + case IB_CM_MRA_REQ_SENT: + case IB_CM_REP_RCVD: case IB_CM_MRA_REP_SENT: case IB_CM_REP_SENT: diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 8187349fb2..9b036706f4 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -760,22 +760,6 @@ static int cma_verify_rep(struct rdma_id_private *id_priv, void *data) return 0; } -static int cma_rtu_recv(struct rdma_id_private *id_priv) -{ - int ret; - - ret = cma_modify_qp_rts(&id_priv->id); - if (ret) - goto reject; - - return 0; -reject: - cma_modify_qp_err(&id_priv->id); - ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, - NULL, 0, NULL, 0); - return ret; -} - static void cma_set_rep_event_data(struct rdma_cm_event *event, struct ib_cm_rep_event_param *rep_data, void *private_data) @@ -821,9 +805,8 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) ib_event->private_data); break; case IB_CM_RTU_RECEIVED: - event.status = cma_rtu_recv(id_priv); - event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR : - RDMA_CM_EVENT_ESTABLISHED; + case IB_CM_USER_ESTABLISHED: + event.event = RDMA_CM_EVENT_ESTABLISHED; break; case IB_CM_DREQ_ERROR: event.status = -ETIMEDOUT; /* fall through */ @@ -1989,11 +1972,25 @@ static int cma_accept_ib(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct ib_cm_rep_param rep; - int ret; + struct ib_qp_attr qp_attr; + int qp_attr_mask, ret; - ret = cma_modify_qp_rtr(&id_priv->id); - if (ret) - return ret; + if (id_priv->id.qp) { + ret = cma_modify_qp_rtr(&id_priv->id); + if (ret) + goto out; + + qp_attr.qp_state = IB_QPS_RTS; + ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, &qp_attr, + &qp_attr_mask); + if (ret) + goto out; + + qp_attr.max_rd_atomic = conn_param->initiator_depth; + ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); + if (ret) + goto out; + } memset(&rep, 0, sizeof rep); rep.qp_num = id_priv->qp_num; @@ -2008,7 +2005,9 @@ static int cma_accept_ib(struct rdma_id_private *id_priv, rep.rnr_retry_count = conn_param->rnr_retry_count; rep.srq = id_priv->srq ? 1 : 0; - return ib_send_cm_rep(id_priv->cm_id.ib, &rep); + ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep); +out: + return ret; } static int cma_accept_iw(struct rdma_id_private *id_priv, @@ -2073,6 +2072,27 @@ reject: } EXPORT_SYMBOL(rdma_accept); +int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event) +{ + struct rdma_id_private *id_priv; + int ret; + + id_priv = container_of(id, struct rdma_id_private, id); + if (!cma_comp(id_priv, CMA_CONNECT)) + return -EINVAL; + + switch (id->device->node_type) { + case RDMA_NODE_IB_CA: + ret = ib_cm_notify(id_priv->cm_id.ib, event); + break; + default: + ret = 0; + break; + } + return ret; +} +EXPORT_SYMBOL(rdma_notify); + int rdma_reject(struct rdma_cm_id *id, const void *private_data, u8 private_data_len) { diff --git a/include/rdma/rdma_cm.h b/include/rdma/rdma_cm.h index aa6ce47537..6b8ec486d0 100644 --- a/include/rdma/rdma_cm.h +++ b/include/rdma/rdma_cm.h @@ -252,6 +252,21 @@ int rdma_listen(struct rdma_cm_id *id, int backlog); */ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param); +/** + * rdma_notify - Notifies the RDMA CM of an asynchronous event that has + * occurred on the connection. + * @id: Connection identifier to transition to established. + * @event: Asynchronous event. + * + * This routine should be invoked by users to notify the CM of relevant + * communication events. Events that should be reported to the CM and + * when to report them are: + * + * IB_EVENT_COMM_EST - Used when a message is received on a connected + * QP before an RTU has been received. + */ +int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event); + /** * rdma_reject - Called to reject a connection request or response. */ -- 2.39.5