thp: improve order in lru list for split huge page

Put the tail subpages of an isolated hugepage under splitting in the
lru reclaim head as they supposedly should be isolated too next.

Queues the subpages in physical order in the lru for non isolated
hugepages under splitting. That might provide some theoretical cache
benefit to the buddy allocator later.

Signed-off-by: Shaohua Li <shaohua.li@intel.com>
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Johannes Weiner <jweiner@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 5a595554bd8c842d930890fdf75168535fbec493..76cc3f7dd4f057cd44540edf3dfd5d479e8e31ca 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1228,7 +1228,6 @@ static int __split_huge_page_splitting(struct page *page,
 static void __split_huge_page_refcount(struct page *page)
 {
        int i;
-       unsigned long head_index = page->index;
        struct zone *zone = page_zone(page);
        int zonestat;
        int tail_count = 0;
@@ -1239,7 +1238,7 @@ static void __split_huge_page_refcount(struct page *page)
        /* complete memcg works before add pages to LRU */
        mem_cgroup_split_huge_fixup(page);
 
-       for (i = 1; i < HPAGE_PMD_NR; i++) {
+       for (i = HPAGE_PMD_NR - 1; i >= 1; i--) {
                struct page *page_tail = page + i;
 
                /* tail_page->_mapcount cannot change */
@@ -1302,7 +1301,7 @@ static void __split_huge_page_refcount(struct page *page)
                BUG_ON(page_tail->mapping);
                page_tail->mapping = page->mapping;
 
-               page_tail->index = ++head_index;
+               page_tail->index = page->index + i;
 
                BUG_ON(!PageAnon(page_tail));
                BUG_ON(!PageUptodate(page_tail));

diff --git a/mm/swap.c b/mm/swap.c
index 126da2919f60863858511d909280c8ceaf9c08c0..ddccf8e0b4ae597084742c661dd6ef60a8167eea 100644 (file)
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -681,7 +681,7 @@ void lru_add_page_tail(struct zone* zone,
                if (likely(PageLRU(page)))
                        list_add(&page_tail->lru, page->lru.prev);
                else
-                       list_add(&page_tail->lru, &lruvec->lists[lru]);
+                       list_add(&page_tail->lru, lruvec->lists[lru].prev);
                __mod_zone_page_state(zone, NR_LRU_BASE + lru,
                                      hpage_nr_pages(page_tail));
        } else {