mm: vmscan: scan until it finds eligible pages

[karo-tx-linux.git] / mm / vmscan.c

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 4e7ed65842aff72beb90815262dacd29987cbcf8..8ad39bbc79e67eaff24c42201ae2470ffc21d0a7 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1449,7 +1449,7 @@ static __always_inline void update_lru_sizes(struct lruvec *lruvec,
  *
  * Appropriate locks must be held before calling this function.
  *
- * @nr_to_scan:        The number of pages to look through on the list.
+ * @nr_to_scan:        The number of eligible pages to look through on the list.
  * @lruvec:    The LRU vector to pull pages from.
  * @dst:       The temp list to put pages on to.
  * @nr_scanned:        The number of pages that were scanned.
@@ -1469,11 +1469,13 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
        unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 };
        unsigned long nr_skipped[MAX_NR_ZONES] = { 0, };
        unsigned long skipped = 0;
-       unsigned long scan, nr_pages;
+       unsigned long scan, total_scan, nr_pages;
        LIST_HEAD(pages_skipped);
 
-       for (scan = 0; scan < nr_to_scan && nr_taken < nr_to_scan &&
-                                       !list_empty(src); scan++) {
+       scan = 0;
+       for (total_scan = 0;
+            scan < nr_to_scan && nr_taken < nr_to_scan && !list_empty(src);
+            total_scan++) {
                struct page *page;
 
                page = lru_to_page(src);
@@ -1487,6 +1489,13 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
                        continue;
                }
 
+               /*
+                * Do not count skipped pages because that makes the function
+                * return with no isolated pages if the LRU mostly contains
+                * ineligible pages.  This causes the VM to not reclaim any
+                * pages, triggering a premature OOM.
+                */
+               scan++;
                switch (__isolate_lru_page(page, mode)) {
                case 0:
                        nr_pages = hpage_nr_pages(page);
@@ -1524,9 +1533,9 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
                        skipped += nr_skipped[zid];
                }
        }
-       *nr_scanned = scan;
+       *nr_scanned = total_scan;
        trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
-                                   scan, skipped, nr_taken, mode, lru);
+                                   total_scan, skipped, nr_taken, mode, lru);
        update_lru_sizes(lruvec, lru, nr_zone_taken);
        return nr_taken;
 }
@@ -3036,6 +3045,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
        struct zonelist *zonelist;
        unsigned long nr_reclaimed;
        int nid;
+       unsigned int noreclaim_flag;
        struct scan_control sc = {
                .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
                .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) |
@@ -3062,9 +3072,9 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
                                            sc.gfp_mask,
                                            sc.reclaim_idx);
 
-       current->flags |= PF_MEMALLOC;
+       noreclaim_flag = memalloc_noreclaim_save();
        nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
-       current->flags &= ~PF_MEMALLOC;
+       memalloc_noreclaim_restore(noreclaim_flag);
 
        trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
 
@@ -3589,8 +3599,9 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
        struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
        struct task_struct *p = current;
        unsigned long nr_reclaimed;
+       unsigned int noreclaim_flag;
 
-       p->flags |= PF_MEMALLOC;
+       noreclaim_flag = memalloc_noreclaim_save();
        lockdep_set_current_reclaim_state(sc.gfp_mask);
        reclaim_state.reclaimed_slab = 0;
        p->reclaim_state = &reclaim_state;
@@ -3599,7 +3610,7 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
 
        p->reclaim_state = NULL;
        lockdep_clear_current_reclaim_state();
-       p->flags &= ~PF_MEMALLOC;
+       memalloc_noreclaim_restore(noreclaim_flag);
 
        return nr_reclaimed;
 }
@@ -3764,6 +3775,7 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
        struct task_struct *p = current;
        struct reclaim_state reclaim_state;
        int classzone_idx = gfp_zone(gfp_mask);
+       unsigned int noreclaim_flag;
        struct scan_control sc = {
                .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
                .gfp_mask = (gfp_mask = current_gfp_context(gfp_mask)),
@@ -3781,7 +3793,8 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
         * and we also need to be able to write out pages for RECLAIM_WRITE
         * and RECLAIM_UNMAP.
         */
-       p->flags |= PF_MEMALLOC | PF_SWAPWRITE;
+       noreclaim_flag = memalloc_noreclaim_save();
+       p->flags |= PF_SWAPWRITE;
        lockdep_set_current_reclaim_state(gfp_mask);
        reclaim_state.reclaimed_slab = 0;
        p->reclaim_state = &reclaim_state;
@@ -3797,7 +3810,8 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
        }
 
        p->reclaim_state = NULL;
-       current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE);
+       current->flags &= ~PF_SWAPWRITE;
+       memalloc_noreclaim_restore(noreclaim_flag);
        lockdep_clear_current_reclaim_state();
        return sc.nr_reclaimed >= nr_pages;
 }