xprtrdma: Replace global lkey with lkey local to PD

The core API has changed so that devices that do not have a global
DMA lkey automatically create an mr, per-PD, and make that lkey
available. The global DMA lkey interface is going away in favor of
the per-PD DMA lkey.

The per-PD DMA lkey is always available. Convert xprtrdma to use the
device's per-PD DMA lkey for regbufs, no matter which memory
registration scheme is in use.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Sagi Grimberg <sagig@mellanox.com>
Cc: linux-nfs <linux-nfs@vger.kernel.org>
Acked-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/net/sunrpc/xprtrdma/fmr_ops.c b/net/sunrpc/xprtrdma/fmr_ops.c
index cb25c89da6239154475d6c31736e328d13f19134..f1e8dafbd5079b3406a769ba4854ecba229edca6 100644 (file)
--- a/net/sunrpc/xprtrdma/fmr_ops.c
+++ b/net/sunrpc/xprtrdma/fmr_ops.c
@@ -39,25 +39,6 @@ static int
 fmr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
            struct rpcrdma_create_data_internal *cdata)
 {
-       struct ib_device_attr *devattr = &ia->ri_devattr;
-       struct ib_mr *mr;
-
-       /* Obtain an lkey to use for the regbufs, which are
-        * protected from remote access.
-        */
-       if (devattr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) {
-               ia->ri_dma_lkey = ia->ri_device->local_dma_lkey;
-       } else {
-               mr = ib_get_dma_mr(ia->ri_pd, IB_ACCESS_LOCAL_WRITE);
-               if (IS_ERR(mr)) {
-                       pr_err("%s: ib_get_dma_mr for failed with %lX\n",
-                              __func__, PTR_ERR(mr));
-                       return -ENOMEM;
-               }
-               ia->ri_dma_lkey = ia->ri_dma_mr->lkey;
-               ia->ri_dma_mr = mr;
-       }
-
        return 0;
 }
 

diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c
index d6653f5d0830378cd08531afb61c0b766ae8b6b9..5318951b3b531ca322f1a0c3639a9079d3599555 100644 (file)
--- a/net/sunrpc/xprtrdma/frwr_ops.c
+++ b/net/sunrpc/xprtrdma/frwr_ops.c
@@ -189,11 +189,6 @@ frwr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
        struct ib_device_attr *devattr = &ia->ri_devattr;
        int depth, delta;
 
-       /* Obtain an lkey to use for the regbufs, which are
-        * protected from remote access.
-        */
-       ia->ri_dma_lkey = ia->ri_device->local_dma_lkey;
-
        ia->ri_max_frmr_depth =
                        min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS,
                              devattr->max_fast_reg_page_list_len);

diff --git a/net/sunrpc/xprtrdma/physical_ops.c b/net/sunrpc/xprtrdma/physical_ops.c
index 72cf8b15bbb4e331d49f937c58abd85f7dd70862..617b76f22154c41b41cdccd329cbbe9f00a3fabe 100644 (file)
--- a/net/sunrpc/xprtrdma/physical_ops.c
+++ b/net/sunrpc/xprtrdma/physical_ops.c
@@ -23,7 +23,6 @@ static int
 physical_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
                 struct rpcrdma_create_data_internal *cdata)
 {
-       struct ib_device_attr *devattr = &ia->ri_devattr;
        struct ib_mr *mr;
 
        /* Obtain an rkey to use for RPC data payloads.
@@ -37,15 +36,8 @@ physical_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
                       __func__, PTR_ERR(mr));
                return -ENOMEM;
        }
-       ia->ri_dma_mr = mr;
-
-       /* Obtain an lkey to use for regbufs.
-        */
-       if (devattr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY)
-               ia->ri_dma_lkey = ia->ri_device->local_dma_lkey;
-       else
-               ia->ri_dma_lkey = ia->ri_dma_mr->lkey;
 
+       ia->ri_dma_mr = mr;
        return 0;
 }
 

diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 682996779970c6ccae749c9de566f06a9b205c80..eb081ad05e33bb65a89b4afb499177dff4d2de89 100644 (file)
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -1252,7 +1252,7 @@ rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags)
                goto out_free;
 
        iov->length = size;
-       iov->lkey = ia->ri_dma_lkey;
+       iov->lkey = ia->ri_pd->local_dma_lkey;
        rb->rg_size = size;
        rb->rg_owner = NULL;
        return rb;

diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index 02512221b8bc885dde93b987c561b344f6e96722..c09414e6f91b0428bd7cc5fd6f2b1c67f8830b23 100644 (file)
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -65,7 +65,6 @@ struct rpcrdma_ia {
        struct rdma_cm_id       *ri_id;
        struct ib_pd            *ri_pd;
        struct ib_mr            *ri_dma_mr;
-       u32                     ri_dma_lkey;
        struct completion       ri_done;
        int                     ri_async_rc;
        unsigned int            ri_max_frmr_depth;