mm: vmstat: move slab statistics from zone to node counters

[karo-tx-linux.git] / mm / vmscan.c

diff --git a/mm/vmscan.c b/mm/vmscan.c
index c3c1c6ac62da67f233b4c146505e35e919ea97a9..9e95fafc026b4174331aee5b8dd91f0ba099a8c4 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -708,7 +708,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page,
                mem_cgroup_swapout(page, swap);
                __delete_from_swap_cache(page);
                spin_unlock_irqrestore(&mapping->tree_lock, flags);
-               swapcache_free(swap);
+               put_swap_page(page, swap);
        } else {
                void (*freepage)(struct page *);
                void *shadow = NULL;
@@ -1125,8 +1125,36 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                    !PageSwapCache(page)) {
                        if (!(sc->gfp_mask & __GFP_IO))
                                goto keep_locked;
-                       if (!add_to_swap(page, page_list))
+                       if (PageTransHuge(page)) {
+                               /* cannot split THP, skip it */
+                               if (!can_split_huge_page(page, NULL))
+                                       goto activate_locked;
+                               /*
+                                * Split pages without a PMD map right
+                                * away. Chances are some or all of the
+                                * tail pages can be freed without IO.
+                                */
+                               if (!compound_mapcount(page) &&
+                                   split_huge_page_to_list(page, page_list))
+                                       goto activate_locked;
+                       }
+                       if (!add_to_swap(page)) {
+                               if (!PageTransHuge(page))
+                                       goto activate_locked;
+                               /* Split THP and swap individual base pages */
+                               if (split_huge_page_to_list(page, page_list))
+                                       goto activate_locked;
+                               if (!add_to_swap(page))
+                                       goto activate_locked;
+                       }
+
+                       /* XXX: We don't support THP writes */
+                       if (PageTransHuge(page) &&
+                                 split_huge_page_to_list(page, page_list)) {
+                               delete_from_swap_cache(page);
                                goto activate_locked;
+                       }
+
                        may_enter_fs = 1;
 
                        /* Adding to swap updated mapping */
@@ -1266,6 +1294,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                        }
 
                        count_vm_event(PGLAZYFREED);
+                       count_memcg_page_event(page, PGLAZYFREED);
                } else if (!mapping || !__remove_mapping(mapping, page, true))
                        goto keep_locked;
                /*
@@ -1295,6 +1324,7 @@ activate_locked:
                if (!PageMlocked(page)) {
                        SetPageActive(page);
                        pgactivate++;
+                       count_memcg_page_event(page, PGACTIVATE);
                }
 keep_locked:
                unlock_page(page);
@@ -1734,11 +1764,16 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
        __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
        reclaim_stat->recent_scanned[file] += nr_taken;
 
-       if (global_reclaim(sc)) {
-               if (current_is_kswapd())
+       if (current_is_kswapd()) {
+               if (global_reclaim(sc))
                        __count_vm_events(PGSCAN_KSWAPD, nr_scanned);
-               else
+               count_memcg_events(lruvec_memcg(lruvec), PGSCAN_KSWAPD,
+                                  nr_scanned);
+       } else {
+               if (global_reclaim(sc))
                        __count_vm_events(PGSCAN_DIRECT, nr_scanned);
+               count_memcg_events(lruvec_memcg(lruvec), PGSCAN_DIRECT,
+                                  nr_scanned);
        }
        spin_unlock_irq(&pgdat->lru_lock);
 
@@ -1750,11 +1785,16 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
 
        spin_lock_irq(&pgdat->lru_lock);
 
-       if (global_reclaim(sc)) {
-               if (current_is_kswapd())
+       if (current_is_kswapd()) {
+               if (global_reclaim(sc))
                        __count_vm_events(PGSTEAL_KSWAPD, nr_reclaimed);
-               else
+               count_memcg_events(lruvec_memcg(lruvec), PGSTEAL_KSWAPD,
+                                  nr_reclaimed);
+       } else {
+               if (global_reclaim(sc))
                        __count_vm_events(PGSTEAL_DIRECT, nr_reclaimed);
+               count_memcg_events(lruvec_memcg(lruvec), PGSTEAL_DIRECT,
+                                  nr_reclaimed);
        }
 
        putback_inactive_pages(lruvec, &page_list);
@@ -1899,8 +1939,11 @@ static unsigned move_active_pages_to_lru(struct lruvec *lruvec,
                }
        }
 
-       if (!is_active_lru(lru))
+       if (!is_active_lru(lru)) {
                __count_vm_events(PGDEACTIVATE, nr_moved);
+               count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
+                                  nr_moved);
+       }
 
        return nr_moved;
 }
@@ -1938,6 +1981,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
        reclaim_stat->recent_scanned[file] += nr_taken;
 
        __count_vm_events(PGREFILL, nr_scanned);
+       count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
 
        spin_unlock_irq(&pgdat->lru_lock);
 
@@ -2967,7 +3011,7 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
        unsigned long nr_reclaimed;
        struct scan_control sc = {
                .nr_to_reclaim = SWAP_CLUSTER_MAX,
-               .gfp_mask = (gfp_mask = current_gfp_context(gfp_mask)),
+               .gfp_mask = current_gfp_context(gfp_mask),
                .reclaim_idx = gfp_zone(gfp_mask),
                .order = order,
                .nodemask = nodemask,
@@ -2982,12 +3026,12 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
         * 1 is returned so that the page allocator does not OOM kill at this
         * point.
         */
-       if (throttle_direct_reclaim(gfp_mask, zonelist, nodemask))
+       if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask))
                return 1;
 
        trace_mm_vmscan_direct_reclaim_begin(order,
                                sc.may_writepage,
-                               gfp_mask,
+                               sc.gfp_mask,
                                sc.reclaim_idx);
 
        nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
@@ -3774,17 +3818,16 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
        const unsigned long nr_pages = 1 << order;
        struct task_struct *p = current;
        struct reclaim_state reclaim_state;
-       int classzone_idx = gfp_zone(gfp_mask);
        unsigned int noreclaim_flag;
        struct scan_control sc = {
                .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
-               .gfp_mask = (gfp_mask = current_gfp_context(gfp_mask)),
+               .gfp_mask = current_gfp_context(gfp_mask),
                .order = order,
                .priority = NODE_RECLAIM_PRIORITY,
                .may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE),
                .may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP),
                .may_swap = 1,
-               .reclaim_idx = classzone_idx,
+               .reclaim_idx = gfp_zone(gfp_mask),
        };
 
        cond_resched();
@@ -3795,7 +3838,7 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
         */
        noreclaim_flag = memalloc_noreclaim_save();
        p->flags |= PF_SWAPWRITE;
-       lockdep_set_current_reclaim_state(gfp_mask);
+       lockdep_set_current_reclaim_state(sc.gfp_mask);
        reclaim_state.reclaimed_slab = 0;
        p->reclaim_state = &reclaim_state;
 
@@ -3831,7 +3874,7 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
         * unmapped file backed pages.
         */
        if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
-           sum_zone_node_page_state(pgdat->node_id, NR_SLAB_RECLAIMABLE) <= pgdat->min_slab_pages)
+           node_page_state(pgdat, NR_SLAB_RECLAIMABLE) <= pgdat->min_slab_pages)
                return NODE_RECLAIM_FULL;
 
        /*