thp: transparent hugepage vmstat

[karo-tx-linux.git] / mm / huge_memory.c

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 0c1e8f939f7cffe05215b42da396cd00680e7aa8..7101112a54296b311bfe3723630237e4a138290a 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -233,6 +233,7 @@ static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
        VM_BUG_ON(!PageCompound(page));
        pgtable = pte_alloc_one(mm, haddr);
        if (unlikely(!pgtable)) {
+               mem_cgroup_uncharge_page(page);
                put_page(page);
                return VM_FAULT_OOM;
        }
@@ -243,6 +244,7 @@ static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
        spin_lock(&mm->page_table_lock);
        if (unlikely(!pmd_none(*pmd))) {
                spin_unlock(&mm->page_table_lock);
+               mem_cgroup_uncharge_page(page);
                put_page(page);
                pte_free(mm, pgtable);
        } else {
@@ -286,6 +288,10 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                page = alloc_hugepage(transparent_hugepage_defrag(vma));
                if (unlikely(!page))
                        goto out;
+               if (unlikely(mem_cgroup_newpage_charge(page, mm, GFP_KERNEL))) {
+                       put_page(page);
+                       goto out;
+               }
 
                return __do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page);
        }
@@ -402,9 +408,17 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
        for (i = 0; i < HPAGE_PMD_NR; i++) {
                pages[i] = alloc_page_vma(GFP_HIGHUSER_MOVABLE,
                                          vma, address);
-               if (unlikely(!pages[i])) {
-                       while (--i >= 0)
+               if (unlikely(!pages[i] ||
+                            mem_cgroup_newpage_charge(pages[i], mm,
+                                                      GFP_KERNEL))) {
+                       if (pages[i])
                                put_page(pages[i]);
+                       mem_cgroup_uncharge_start();
+                       while (--i >= 0) {
+                               mem_cgroup_uncharge_page(pages[i]);
+                               put_page(pages[i]);
+                       }
+                       mem_cgroup_uncharge_end();
                        kfree(pages);
                        ret |= VM_FAULT_OOM;
                        goto out;
@@ -455,8 +469,12 @@ out:
 
 out_free_pages:
        spin_unlock(&mm->page_table_lock);
-       for (i = 0; i < HPAGE_PMD_NR; i++)
+       mem_cgroup_uncharge_start();
+       for (i = 0; i < HPAGE_PMD_NR; i++) {
+               mem_cgroup_uncharge_page(pages[i]);
                put_page(pages[i]);
+       }
+       mem_cgroup_uncharge_end();
        kfree(pages);
        goto out;
 }
@@ -501,14 +519,22 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
                goto out;
        }
 
+       if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) {
+               put_page(new_page);
+               put_page(page);
+               ret |= VM_FAULT_OOM;
+               goto out;
+       }
+
        copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
        __SetPageUptodate(new_page);
 
        spin_lock(&mm->page_table_lock);
        put_page(page);
-       if (unlikely(!pmd_same(*pmd, orig_pmd)))
+       if (unlikely(!pmd_same(*pmd, orig_pmd))) {
+               mem_cgroup_uncharge_page(new_page);
                put_page(new_page);
-       else {
+       } else {
                pmd_t entry;
                VM_BUG_ON(!PageHead(page));
                entry = mk_pmd(new_page, vma->vm_page_prot);
@@ -725,6 +751,9 @@ static void __split_huge_page_refcount(struct page *page)
                lru_add_page_tail(zone, page, page_tail);
        }
 
+       __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
+       __mod_zone_page_state(zone, NR_ANON_PAGES, HPAGE_PMD_NR);
+
        ClearPageCompound(page);
        compound_unlock(page);
        spin_unlock_irq(&zone->lru_lock);
@@ -841,6 +870,19 @@ static void __split_huge_page(struct page *page,
                        continue;
                mapcount += __split_huge_page_splitting(page, vma, addr);
        }
+       /*
+        * It is critical that new vmas are added to the tail of the
+        * anon_vma list. This guarantes that if copy_huge_pmd() runs
+        * and establishes a child pmd before
+        * __split_huge_page_splitting() freezes the parent pmd (so if
+        * we fail to prevent copy_huge_pmd() from running until the
+        * whole __split_huge_page() is complete), we will still see
+        * the newly established pmd of the child later during the
+        * walk, to be able to set it as pmd_trans_splitting too.
+        */
+       if (mapcount != page_mapcount(page))
+               printk(KERN_ERR "mapcount %d page_mapcount %d\n",
+                      mapcount, page_mapcount(page));
        BUG_ON(mapcount != page_mapcount(page));
 
        __split_huge_page_refcount(page);
@@ -854,6 +896,9 @@ static void __split_huge_page(struct page *page,
                        continue;
                mapcount2 += __split_huge_page_map(page, vma, addr);
        }
+       if (mapcount != mapcount2)
+               printk(KERN_ERR "mapcount %d mapcount2 %d page_mapcount %d\n",
+                      mapcount, mapcount2, page_mapcount(page));
        BUG_ON(mapcount != mapcount2);
 }
 
@@ -880,6 +925,22 @@ out:
        return ret;
 }
 
+int hugepage_madvise(unsigned long *vm_flags)
+{
+       /*
+        * Be somewhat over-protective like KSM for now!
+        */
+       if (*vm_flags & (VM_HUGEPAGE | VM_SHARED  | VM_MAYSHARE   |
+                        VM_PFNMAP   | VM_IO      | VM_DONTEXPAND |
+                        VM_RESERVED | VM_HUGETLB | VM_INSERTPAGE |
+                        VM_MIXEDMAP | VM_SAO))
+               return -EINVAL;
+
+       *vm_flags |= VM_HUGEPAGE;
+
+       return 0;
+}
+
 void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd)
 {
        struct page *page;