RDMA/bnxt_re: Add HW workaround for avoiding stall for UD QPs

HW stalls out after 0x800000 WQEs are posted for UD QPs.
To workaround this problem, driver will send a modify_qp cmd
to the HW at around the halfway mark(0x400000) so that FW
can accordingly modify the QP context in the HW to prevent this
stall.
This workaround needs to be done for UD, QP1 and Raw Ethertype
packets. Added a counter to keep track of WQEs posted during post_send.

Signed-off-by: Somnath Kotur <somnath.kotur@broadcom.com>
Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
index ebf7be8d4139b87dd8232568603ab1aa8c19d045..d5e457ee7e7b931a4d89b01d508e88d99536254f 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
+++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
@@ -56,6 +56,8 @@
 #define BNXT_RE_MAX_SRQC_COUNT         (64 * 1024)
 #define BNXT_RE_MAX_CQ_COUNT           (64 * 1024)
 
+#define BNXT_RE_UD_QP_HW_STALL         0x400000
+
 struct bnxt_re_work {
        struct work_struct      work;
        unsigned long           event;

diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index d94b1b3041359c5f06e9c29aae4fe8bf57934dca..08e7e59df28c620f0a596b0e57cc4a20822f1296 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -2075,6 +2075,22 @@ static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
        return payload_sz;
 }
 
+static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
+{
+       if ((qp->ib_qp.qp_type == IB_QPT_UD ||
+            qp->ib_qp.qp_type == IB_QPT_GSI ||
+            qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
+            qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
+               int qp_attr_mask;
+               struct ib_qp_attr qp_attr;
+
+               qp_attr_mask = IB_QP_STATE;
+               qp_attr.qp_state = IB_QPS_RTS;
+               bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL);
+               qp->qplib_qp.wqe_cnt = 0;
+       }
+}
+
 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
                                       struct bnxt_re_qp *qp,
                                struct ib_send_wr *wr)
@@ -2120,6 +2136,7 @@ bad:
                wr = wr->next;
        }
        bnxt_qplib_post_send_db(&qp->qplib_qp);
+       bnxt_ud_qp_hw_stall_workaround(qp);
        spin_unlock_irqrestore(&qp->sq_lock, flags);
        return rc;
 }
@@ -2216,6 +2233,7 @@ bad:
                wr = wr->next;
        }
        bnxt_qplib_post_send_db(&qp->qplib_qp);
+       bnxt_ud_qp_hw_stall_workaround(qp);
        spin_unlock_irqrestore(&qp->sq_lock, flags);
 
        return rc;

diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 31593fdf6365a08dce71ea32468cd1b028b7ec3b..f05500bcdcf1e35d21a88aba70b013d0d0d8a2cf 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -1298,6 +1298,9 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
        }
 
        sq->hwq.prod++;
+
+       qp->wqe_cnt++;
+
 done:
        return rc;
 }

diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
index 71539ea7f421de8e0bd480de7cd5a8932e46dcdd..36b7b7db0e3f9782104ee8b049e674edb830226b 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
@@ -250,6 +250,7 @@ struct bnxt_qplib_qp {
        u8                              timeout;
        u8                              retry_cnt;
        u8                              rnr_retry;
+       u64                             wqe_cnt;
        u32                             min_rnr_timer;
        u32                             max_rd_atomic;
        u32                             max_dest_rd_atomic;