tcp: fix/cleanup inet_ehash_locks_alloc()

If tcp ehash table is constrained to a very small number of buckets
(eg boot parameter thash_entries=128), then we can crash if spinlock
array has more entries.

While we are at it, un-inline inet_ehash_locks_alloc() and make
following changes :

- Budget 2 cache lines per cpu worth of 'spinlocks'
- Try to kmalloc() the array to avoid extra TLB pressure.
  (Most servers at Google allocate 8192 bytes for this hash table)
- Get rid of various #ifdef

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h
index 774d24151d4a5bd2496ab2977f46a036e4179086..b73c88a19dd408f0de41f87c80242816fac4b19d 100644 (file)
--- a/include/net/inet_hashtables.h
+++ b/include/net/inet_hashtables.h
@@ -24,7 +24,6 @@
 #include <linux/spinlock.h>
 #include <linux/types.h>
 #include <linux/wait.h>
-#include <linux/vmalloc.h>
 
 #include <net/inet_connection_sock.h>
 #include <net/inet_sock.h>
@@ -164,52 +163,12 @@ static inline spinlock_t *inet_ehash_lockp(
        return &hashinfo->ehash_locks[hash & hashinfo->ehash_locks_mask];
 }
 
-static inline int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo)
-{
-       unsigned int i, size = 256;
-#if defined(CONFIG_PROVE_LOCKING)
-       unsigned int nr_pcpus = 2;
-#else
-       unsigned int nr_pcpus = num_possible_cpus();
-#endif
-       if (nr_pcpus >= 4)
-               size = 512;
-       if (nr_pcpus >= 8)
-               size = 1024;
-       if (nr_pcpus >= 16)
-               size = 2048;
-       if (nr_pcpus >= 32)
-               size = 4096;
-       if (sizeof(spinlock_t) != 0) {
-#ifdef CONFIG_NUMA
-               if (size * sizeof(spinlock_t) > PAGE_SIZE)
-                       hashinfo->ehash_locks = vmalloc(size * sizeof(spinlock_t));
-               else
-#endif
-               hashinfo->ehash_locks = kmalloc(size * sizeof(spinlock_t),
-                                               GFP_KERNEL);
-               if (!hashinfo->ehash_locks)
-                       return ENOMEM;
-               for (i = 0; i < size; i++)
-                       spin_lock_init(&hashinfo->ehash_locks[i]);
-       }
-       hashinfo->ehash_locks_mask = size - 1;
-       return 0;
-}
+int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo);
 
 static inline void inet_ehash_locks_free(struct inet_hashinfo *hashinfo)
 {
-       if (hashinfo->ehash_locks) {
-#ifdef CONFIG_NUMA
-               unsigned int size = (hashinfo->ehash_locks_mask + 1) *
-                                                       sizeof(spinlock_t);
-               if (size > PAGE_SIZE)
-                       vfree(hashinfo->ehash_locks);
-               else
-#endif
-               kfree(hashinfo->ehash_locks);
-               hashinfo->ehash_locks = NULL;
-       }
+       kvfree(hashinfo->ehash_locks);
+       hashinfo->ehash_locks = NULL;
 }
 
 struct inet_bind_bucket *

diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
index 3766bddb3e8a7303123aa7e32507f6f7801c10d5..185efef0f1251ba9d45fabb3ed51777a8be097a6 100644 (file)
--- a/net/ipv4/inet_hashtables.c
+++ b/net/ipv4/inet_hashtables.c
@@ -18,6 +18,7 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/wait.h>
+#include <linux/vmalloc.h>
 
 #include <net/inet_connection_sock.h>
 #include <net/inet_hashtables.h>
@@ -609,3 +610,33 @@ void inet_hashinfo_init(struct inet_hashinfo *h)
                }
 }
 EXPORT_SYMBOL_GPL(inet_hashinfo_init);
+
+int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo)
+{
+       unsigned int i, nblocks = 1;
+
+       if (sizeof(spinlock_t) != 0) {
+               /* allocate 2 cache lines or at least one spinlock per cpu */
+               nblocks = max_t(unsigned int,
+                               2 * L1_CACHE_BYTES / sizeof(spinlock_t),
+                               1);
+               nblocks = roundup_pow_of_two(nblocks * num_possible_cpus());
+
+               /* no more locks than number of hash buckets */
+               nblocks = min(nblocks, hashinfo->ehash_mask + 1);
+
+               hashinfo->ehash_locks = kmalloc_array(nblocks, sizeof(spinlock_t),
+                                                     GFP_KERNEL | __GFP_NOWARN);
+               if (!hashinfo->ehash_locks)
+                       hashinfo->ehash_locks = vmalloc(nblocks * sizeof(spinlock_t));
+
+               if (!hashinfo->ehash_locks)
+                       return -ENOMEM;
+
+               for (i = 0; i < nblocks; i++)
+                       spin_lock_init(&hashinfo->ehash_locks[i]);
+       }
+       hashinfo->ehash_locks_mask = nblocks - 1;
+       return 0;
+}
+EXPORT_SYMBOL_GPL(inet_ehash_locks_alloc);