hugetlb: pull gigantic page initialisation out of the default path

commit 18229df5b613ed0732a766fc37850de2e7988e43 upstream

As we can determine exactly when a gigantic page is in use we can optimise
the common regular page cases by pulling out gigantic page initialisation
into its own function.  As gigantic pages are never released to buddy we
do not need a destructor.  This effectivly reverts the previous change to
the main buddy allocator.  It also adds a paranoid check to ensure we
never release gigantic pages from hugetlbfs to the main buddy.

Signed-off-by: Andy Whitcroft <apw@shadowen.org>
Cc: Jon Tollefson <kniht@linux.vnet.ibm.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Lameter <cl@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 67a71191136ed8f6ad9dad03b472440baa322210..9dbda3b5b86d4ce8f0c9e1e282eefb17a307c878 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -455,6 +455,8 @@ static void update_and_free_page(struct hstate *h, struct page *page)
 {
        int i;
 
+       VM_BUG_ON(h->order >= MAX_ORDER);
+
        h->nr_huge_pages--;
        h->nr_huge_pages_node[page_to_nid(page)]--;
        for (i = 0; i < pages_per_huge_page(h); i++) {
@@ -969,6 +971,14 @@ found:
        return 1;
 }
 
+static void prep_compound_huge_page(struct page *page, int order)
+{
+       if (unlikely(order > (MAX_ORDER - 1)))
+               prep_compound_gigantic_page(page, order);
+       else
+               prep_compound_page(page, order);
+}
+
 /* Put bootmem huge pages into the standard lists after mem_map is up */
 static void __init gather_bootmem_prealloc(void)
 {
@@ -979,7 +989,7 @@ static void __init gather_bootmem_prealloc(void)
                struct hstate *h = m->hstate;
                __ClearPageReserved(page);
                WARN_ON(page_count(page) != 1);
-               prep_compound_page(page, h->order);
+               prep_compound_huge_page(page, h->order);
                prep_new_huge_page(h, page, page_to_nid(page));
        }
 }

diff --git a/mm/internal.h b/mm/internal.h
index 1f43f7416972d3c4f1b07ab04898008a8ec76a81..daff45047b14546f6b9e086dbf2beea184553373 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -17,6 +17,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
                unsigned long floor, unsigned long ceiling);
 
 extern void prep_compound_page(struct page *page, unsigned long order);
+extern void prep_compound_gigantic_page(struct page *page, unsigned long order);
 
 static inline void set_page_count(struct page *page, int v)
 {

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 27b8681139fdb71e4d0da204ecc4a65fc3f7a1b5..ed5cdae10bfc2556ccdba39684279dcf34b46449 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -265,6 +265,23 @@ static void free_compound_page(struct page *page)
 }
 
 void prep_compound_page(struct page *page, unsigned long order)
+{
+       int i;
+       int nr_pages = 1 << order;
+
+       set_compound_page_dtor(page, free_compound_page);
+       set_compound_order(page, order);
+       __SetPageHead(page);
+       for (i = 1; i < nr_pages; i++) {
+               struct page *p = page + i;
+
+               __SetPageTail(p);
+               p->first_page = page;
+       }
+}
+
+#ifdef CONFIG_HUGETLBFS
+void prep_compound_gigantic_page(struct page *page, unsigned long order)
 {
        int i;
        int nr_pages = 1 << order;
@@ -273,19 +290,17 @@ void prep_compound_page(struct page *page, unsigned long order)
        set_compound_page_dtor(page, free_compound_page);
        set_compound_order(page, order);
        __SetPageHead(page);
-       for (i = 1; i < nr_pages; i++, p++) {
-               if (unlikely((i & (MAX_ORDER_NR_PAGES - 1)) == 0))
-                       p = pfn_to_page(page_to_pfn(page) + i);
+       for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
                __SetPageTail(p);
                p->first_page = page;
        }
 }
+#endif
 
 static void destroy_compound_page(struct page *page, unsigned long order)
 {
        int i;
        int nr_pages = 1 << order;
-       struct page *p = page + 1;
 
        if (unlikely(compound_order(page) != order))
                bad_page(page);
@@ -293,9 +308,8 @@ static void destroy_compound_page(struct page *page, unsigned long order)
        if (unlikely(!PageHead(page)))
                        bad_page(page);
        __ClearPageHead(page);
-       for (i = 1; i < nr_pages; i++, p++) {
-               if (unlikely((i & (MAX_ORDER_NR_PAGES - 1)) == 0))
-                       p = pfn_to_page(page_to_pfn(page) + i);
+       for (i = 1; i < nr_pages; i++) {
+               struct page *p = page + i;
 
                if (unlikely(!PageTail(p) |
                                (p->first_page != page)))