Merge branch 'akpm-current/current'

[karo-tx-linux.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 545a170ebf9f66cf0e3716c9cd6f4cb7eef0eda6..d2388c850b6534a9467434f1ab1e0876ba5cb7d4 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -985,23 +985,22 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x)
 
 /*
  * Tracking of fully allocated slabs for debugging purposes.
- *
- * list_lock must be held.
  */
 static void add_full(struct kmem_cache *s,
        struct kmem_cache_node *n, struct page *page)
 {
+       lockdep_assert_held(&n->list_lock);
+
        if (!(s->flags & SLAB_STORE_USER))
                return;
 
        list_add(&page->lru, &n->full);
 }
 
-/*
- * list_lock must be held.
- */
-static void remove_full(struct kmem_cache *s, struct page *page)
+static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page)
 {
+       lockdep_assert_held(&n->list_lock);
+
        if (!(s->flags & SLAB_STORE_USER))
                return;
 
@@ -1250,7 +1249,8 @@ static inline int check_object(struct kmem_cache *s, struct page *page,
                        void *object, u8 val) { return 1; }
 static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
                                        struct page *page) {}
-static inline void remove_full(struct kmem_cache *s, struct page *page) {}
+static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
+                                       struct page *page) {}
 static inline unsigned long kmem_cache_flags(unsigned long object_size,
        unsigned long flags, const char *name,
        void (*ctor)(void *))
@@ -1504,12 +1504,12 @@ static void discard_slab(struct kmem_cache *s, struct page *page)
 
 /*
  * Management of partially allocated slabs.
- *
- * list_lock must be held.
  */
 static inline void add_partial(struct kmem_cache_node *n,
                                struct page *page, int tail)
 {
+       lockdep_assert_held(&n->list_lock);
+
        n->nr_partial++;
        if (tail == DEACTIVATE_TO_TAIL)
                list_add_tail(&page->lru, &n->partial);
@@ -1517,12 +1517,11 @@ static inline void add_partial(struct kmem_cache_node *n,
                list_add(&page->lru, &n->partial);
 }
 
-/*
- * list_lock must be held.
- */
 static inline void remove_partial(struct kmem_cache_node *n,
                                        struct page *page)
 {
+       lockdep_assert_held(&n->list_lock);
+
        list_del(&page->lru);
        n->nr_partial--;
 }
@@ -1532,8 +1531,6 @@ static inline void remove_partial(struct kmem_cache_node *n,
  * return the pointer to the freelist.
  *
  * Returns a list of objects or NULL if it fails.
- *
- * Must hold list_lock since we modify the partial list.
  */
 static inline void *acquire_slab(struct kmem_cache *s,
                struct kmem_cache_node *n, struct page *page,
@@ -1543,6 +1540,8 @@ static inline void *acquire_slab(struct kmem_cache *s,
        unsigned long counters;
        struct page new;
 
+       lockdep_assert_held(&n->list_lock);
+
        /*
         * Zap the freelist and set the frozen bit.
         * The old freelist is the list of objects for the
@@ -1559,7 +1558,7 @@ static inline void *acquire_slab(struct kmem_cache *s,
                new.freelist = freelist;
        }
 
-       VM_BUG_ON(new.frozen);
+       VM_BUG_ON_PAGE(new.frozen, &new);
        new.frozen = 1;
 
        if (!__cmpxchg_double_slab(s, page,
@@ -1812,7 +1811,7 @@ static void deactivate_slab(struct kmem_cache *s, struct page *page,
                        set_freepointer(s, freelist, prior);
                        new.counters = counters;
                        new.inuse--;
-                       VM_BUG_ON(!new.frozen);
+                       VM_BUG_ON_PAGE(!new.frozen, &new);
 
                } while (!__cmpxchg_double_slab(s, page,
                        prior, counters,
@@ -1840,7 +1839,7 @@ redo:
 
        old.freelist = page->freelist;
        old.counters = page->counters;
-       VM_BUG_ON(!old.frozen);
+       VM_BUG_ON_PAGE(!old.frozen, &old);
 
        /* Determine target state of the slab */
        new.counters = old.counters;
@@ -1887,7 +1886,7 @@ redo:
 
                else if (l == M_FULL)
 
-                       remove_full(s, page);
+                       remove_full(s, n, page);
 
                if (m == M_PARTIAL) {
 
@@ -1952,7 +1951,7 @@ static void unfreeze_partials(struct kmem_cache *s,
 
                        old.freelist = page->freelist;
                        old.counters = page->counters;
-                       VM_BUG_ON(!old.frozen);
+                       VM_BUG_ON_PAGE(!old.frozen, &old);
 
                        new.counters = old.counters;
                        new.freelist = old.freelist;
@@ -2225,7 +2224,7 @@ static inline void *get_freelist(struct kmem_cache *s, struct page *page)
                counters = page->counters;
 
                new.counters = counters;
-               VM_BUG_ON(!new.frozen);
+               VM_BUG_ON_PAGE(!new.frozen, &new);
 
                new.inuse = page->objects;
                new.frozen = freelist != NULL;
@@ -2319,7 +2318,7 @@ load_freelist:
         * page is pointing to the page from which the objects are obtained.
         * That page must be frozen for per cpu allocations to work.
         */
-       VM_BUG_ON(!c->page->frozen);
+       VM_BUG_ON_PAGE(!c->page->frozen, c->page);
        c->freelist = get_freepointer(s, freelist);
        c->tid = next_tid(c->tid);
        local_irq_restore(flags);
@@ -2541,7 +2540,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
                new.inuse--;
                if ((!new.inuse || !prior) && !was_frozen) {
 
-                       if (kmem_cache_has_cpu_partial(s) && !prior)
+                       if (kmem_cache_has_cpu_partial(s) && !prior) {
 
                                /*
                                 * Slab was on no list before and will be
@@ -2551,7 +2550,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
                                 */
                                new.frozen = 1;
 
-                       else { /* Needs to be taken off a list */
+                       } else { /* Needs to be taken off a list */
 
                                n = get_node(s, page_to_nid(page));
                                /*
@@ -2600,7 +2599,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
         */
        if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
                if (kmem_cache_debug(s))
-                       remove_full(s, page);
+                       remove_full(s, n, page);
                add_partial(n, page, DEACTIVATE_TO_TAIL);
                stat(s, FREE_ADD_PARTIAL);
        }
@@ -2614,9 +2613,10 @@ slab_empty:
                 */
                remove_partial(n, page);
                stat(s, FREE_REMOVE_PARTIAL);
-       } else
+       } else {
                /* Slab must be on the full list */
-               remove_full(s, page);
+               remove_full(s, n, page);
+       }
 
        spin_unlock_irqrestore(&n->list_lock, flags);
        stat(s, FREE_SLAB);
@@ -4299,7 +4299,13 @@ static ssize_t show_slab_objects(struct kmem_cache *s,
 
                        page = ACCESS_ONCE(c->partial);
                        if (page) {
-                               x = page->pobjects;
+                               node = page_to_nid(page);
+                               if (flags & SO_TOTAL)
+                                       WARN_ON_ONCE(1);
+                               else if (flags & SO_OBJECTS)
+                                       WARN_ON_ONCE(1);
+                               else
+                                       x = page->pages;
                                total += x;
                                nodes[node] += x;
                        }
@@ -5163,7 +5169,7 @@ static int sysfs_slab_add(struct kmem_cache *s)
        }
 
        s->kobj.kset = slab_kset;
-       err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, name);
+       err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
        if (err) {
                kobject_put(&s->kobj);
                return err;