NFS: allow lockless access to access_cache

The access cache is used during RCU-walk path lookups, so it is best
to avoid locking if possible as taking a lock kills concurrency.

The rbtree is not rcu-safe and cannot easily be made so.
Instead we simply check the last (i.e. most recent) entry on the LRU
list.  If this doesn't match, then we return -ECHILD and retry in
lock/refcount mode.

This requires freeing the nfs_access_entry struct with rcu, and
requires using rcu access primatives when adding entries to the lru, and
when examining the last entry.

Calling put_rpccred before kfree_rcu looks a bit odd, but as
put_rpccred already provides rcu protection, we know that the cred will
not actually be freed until the next grace period, so any concurrent
access will be safe.

This patch provides about 5% performance improvement on a stat-heavy
synthetic work load with 4 threads on a 2-core CPU.

Signed-off-by: NeilBrown <neilb@suse.de>
Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>

diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c
index 2bfbde0f71769d23b4e52a0621d2e156934e1ceb..1b5f38f48dab6aba4ba5f7e3713785575efc9fec 100644 (file)
--- a/fs/nfs/dir.c
+++ b/fs/nfs/dir.c
@@ -2079,7 +2079,7 @@ MODULE_PARM_DESC(nfs_access_max_cachesize, "NFS access maximum total cache lengt
 static void nfs_access_free_entry(struct nfs_access_entry *entry)
 {
        put_rpccred(entry->cred);
-       kfree(entry);
+       kfree_rcu(entry, rcu_head);
        smp_mb__before_atomic();
        atomic_long_dec(&nfs_access_nr_entries);
        smp_mb__after_atomic();
@@ -2257,6 +2257,38 @@ out_zap:
        return -ENOENT;
 }
 
+static int nfs_access_get_cached_rcu(struct inode *inode, struct rpc_cred *cred, struct nfs_access_entry *res)
+{
+       /* Only check the most recently returned cache entry,
+        * but do it without locking.
+        */
+       struct nfs_inode *nfsi = NFS_I(inode);
+       struct nfs_access_entry *cache;
+       int err = -ECHILD;
+       struct list_head *lh;
+
+       rcu_read_lock();
+       if (nfsi->cache_validity & NFS_INO_INVALID_ACCESS)
+               goto out;
+       lh = rcu_dereference(nfsi->access_cache_entry_lru.prev);
+       cache = list_entry(lh, struct nfs_access_entry, lru);
+       if (lh == &nfsi->access_cache_entry_lru ||
+           cred != cache->cred)
+               cache = NULL;
+       if (cache == NULL)
+               goto out;
+       if (!nfs_have_delegated_attributes(inode) &&
+           !time_in_range_open(jiffies, cache->jiffies, cache->jiffies + nfsi->attrtimeo))
+               goto out;
+       res->jiffies = cache->jiffies;
+       res->cred = cache->cred;
+       res->mask = cache->mask;
+       err = 0;
+out:
+       rcu_read_unlock();
+       return err;
+}
+
 static void nfs_access_add_rbtree(struct inode *inode, struct nfs_access_entry *set)
 {
        struct nfs_inode *nfsi = NFS_I(inode);
@@ -2300,6 +2332,11 @@ void nfs_access_add_cache(struct inode *inode, struct nfs_access_entry *set)
        cache->cred = get_rpccred(set->cred);
        cache->mask = set->mask;
 
+       /* The above field assignments must be visible
+        * before this item appears on the lru.  We cannot easily
+        * use rcu_assign_pointer, so just force the memory barrier.
+        */
+       smp_wmb();
        nfs_access_add_rbtree(inode, cache);
 
        /* Update accounting */
@@ -2339,7 +2376,9 @@ static int nfs_do_access(struct inode *inode, struct rpc_cred *cred, int mask)
 
        trace_nfs_access_enter(inode);
 
-       status = nfs_access_get_cached(inode, cred, &cache);
+       status = nfs_access_get_cached_rcu(inode, cred, &cache);
+       if (status != 0)
+               status = nfs_access_get_cached(inode, cred, &cache);
        if (status == 0)
                goto out_cached;
 

diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
index 60cd9e3779268fc39f1effb96e97addd89f70f9a..5180a7ededecf2797b4cb4a23c1a223f48bbe3cf 100644 (file)
--- a/include/linux/nfs_fs.h
+++ b/include/linux/nfs_fs.h
@@ -52,6 +52,7 @@ struct nfs_access_entry {
        unsigned long           jiffies;
        struct rpc_cred *       cred;
        int                     mask;
+       struct rcu_head         rcu_head;
 };
 
 struct nfs_lockowner {