Merge branch 'i2c/for-current' of git://git.kernel.org/pub/scm/linux/kernel/git/wsa...

[karo-tx-linux.git] / mm / hugetlb.c

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index fd6e0c50f949b5e87b851aa9ccfdeed7a2f17389..a1a0ac0ad6f67ad479916fcbc43036973ddca824 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -887,19 +887,39 @@ static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid)
        return page;
 }
 
-static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
+static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid,
+               nodemask_t *nmask)
 {
-       struct page *page;
-       int node;
+       unsigned int cpuset_mems_cookie;
+       struct zonelist *zonelist;
+       struct zone *zone;
+       struct zoneref *z;
+       int node = -1;
 
-       if (nid != NUMA_NO_NODE)
-               return dequeue_huge_page_node_exact(h, nid);
+       zonelist = node_zonelist(nid, gfp_mask);
+
+retry_cpuset:
+       cpuset_mems_cookie = read_mems_allowed_begin();
+       for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
+               struct page *page;
+
+               if (!cpuset_zone_allowed(zone, gfp_mask))
+                       continue;
+               /*
+                * no need to ask again on the same node. Pool is node rather than
+                * zone aware
+                */
+               if (zone_to_nid(zone) == node)
+                       continue;
+               node = zone_to_nid(zone);
 
-       for_each_online_node(node) {
                page = dequeue_huge_page_node_exact(h, node);
                if (page)
                        return page;
        }
+       if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
+               goto retry_cpuset;
+
        return NULL;
 }
 
@@ -917,15 +937,11 @@ static struct page *dequeue_huge_page_vma(struct hstate *h,
                                unsigned long address, int avoid_reserve,
                                long chg)
 {
-       struct page *page = NULL;
+       struct page *page;
        struct mempolicy *mpol;
-       nodemask_t *nodemask;
        gfp_t gfp_mask;
+       nodemask_t *nodemask;
        int nid;
-       struct zonelist *zonelist;
-       struct zone *zone;
-       struct zoneref *z;
-       unsigned int cpuset_mems_cookie;
 
        /*
         * A child process with MAP_PRIVATE mappings created by their parent
@@ -940,32 +956,15 @@ static struct page *dequeue_huge_page_vma(struct hstate *h,
        if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
                goto err;
 
-retry_cpuset:
-       cpuset_mems_cookie = read_mems_allowed_begin();
        gfp_mask = htlb_alloc_mask(h);
        nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
-       zonelist = node_zonelist(nid, gfp_mask);
-
-       for_each_zone_zonelist_nodemask(zone, z, zonelist,
-                                               MAX_NR_ZONES - 1, nodemask) {
-               if (cpuset_zone_allowed(zone, gfp_mask)) {
-                       page = dequeue_huge_page_node(h, zone_to_nid(zone));
-                       if (page) {
-                               if (avoid_reserve)
-                                       break;
-                               if (!vma_has_reserves(vma, chg))
-                                       break;
-
-                               SetPagePrivate(page);
-                               h->resv_huge_pages--;
-                               break;
-                       }
-               }
+       page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask);
+       if (page && !avoid_reserve && vma_has_reserves(vma, chg)) {
+               SetPagePrivate(page);
+               h->resv_huge_pages--;
        }
 
        mpol_cond_put(mpol);
-       if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
-               goto retry_cpuset;
        return page;
 
 err:
@@ -1385,7 +1384,7 @@ static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
 
        page = __alloc_pages_node(nid,
                htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
-                                               __GFP_REPEAT|__GFP_NOWARN,
+                                               __GFP_RETRY_MAYFAIL|__GFP_NOWARN,
                huge_page_order(h));
        if (page) {
                prep_new_huge_page(h, page, nid);
@@ -1526,7 +1525,7 @@ static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h,
 {
        int order = huge_page_order(h);
 
-       gfp_mask |= __GFP_COMP|__GFP_REPEAT|__GFP_NOWARN;
+       gfp_mask |= __GFP_COMP|__GFP_RETRY_MAYFAIL|__GFP_NOWARN;
        if (nid == NUMA_NO_NODE)
                nid = numa_mem_id();
        return __alloc_pages_nodemask(gfp_mask, order, nid, nmask);
@@ -1633,7 +1632,7 @@ struct page *alloc_huge_page_node(struct hstate *h, int nid)
 
        spin_lock(&hugetlb_lock);
        if (h->free_huge_pages - h->resv_huge_pages > 0)
-               page = dequeue_huge_page_node(h, nid);
+               page = dequeue_huge_page_nodemask(h, gfp_mask, nid, NULL);
        spin_unlock(&hugetlb_lock);
 
        if (!page)
@@ -1642,26 +1641,27 @@ struct page *alloc_huge_page_node(struct hstate *h, int nid)
        return page;
 }
 
-struct page *alloc_huge_page_nodemask(struct hstate *h, nodemask_t *nmask)
+
+struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
+               nodemask_t *nmask)
 {
        gfp_t gfp_mask = htlb_alloc_mask(h);
-       struct page *page = NULL;
-       int node;
 
        spin_lock(&hugetlb_lock);
        if (h->free_huge_pages - h->resv_huge_pages > 0) {
-               for_each_node_mask(node, *nmask) {
-                       page = dequeue_huge_page_node_exact(h, node);
-                       if (page)
-                               break;
+               struct page *page;
+
+               page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask);
+               if (page) {
+                       spin_unlock(&hugetlb_lock);
+                       return page;
                }
        }
        spin_unlock(&hugetlb_lock);
-       if (page)
-               return page;
 
        /* No reservations, try to overcommit */
-       return __alloc_buddy_huge_page(h, gfp_mask, NUMA_NO_NODE, nmask);
+
+       return __alloc_buddy_huge_page(h, gfp_mask, preferred_nid, nmask);
 }
 
 /*
@@ -4078,6 +4078,7 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
        unsigned long vaddr = *position;
        unsigned long remainder = *nr_pages;
        struct hstate *h = hstate_vma(vma);
+       int err = -EFAULT;
 
        while (vaddr < vma->vm_end && remainder) {
                pte_t *pte;
@@ -4154,11 +4155,7 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
                        }
                        ret = hugetlb_fault(mm, vma, vaddr, fault_flags);
                        if (ret & VM_FAULT_ERROR) {
-                               int err = vm_fault_to_errno(ret, flags);
-
-                               if (err)
-                                       return err;
-
+                               err = vm_fault_to_errno(ret, flags);
                                remainder = 0;
                                break;
                        }
@@ -4213,7 +4210,7 @@ same_page:
         */
        *position = vaddr;
 
-       return i ? i : -EFAULT;
+       return i ? i : err;
 }
 
 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE