Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net

[karo-tx-linux.git] / mm / gup.c

diff --git a/mm/gup.c b/mm/gup.c
index fe95a37a417256be4463105832614fbd892106ec..23f01c40c88f63cc88ff62548a4d120b0c406a8e 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -226,6 +226,14 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma,
                        return page;
                return no_page_table(vma, flags);
        }
+       if (is_hugepd(__hugepd(pmd_val(*pmd)))) {
+               page = follow_huge_pd(vma, address,
+                                     __hugepd(pmd_val(*pmd)), flags,
+                                     PMD_SHIFT);
+               if (page)
+                       return page;
+               return no_page_table(vma, flags);
+       }
        if (pmd_devmap(*pmd)) {
                ptl = pmd_lock(mm, pmd);
                page = follow_devmap_pmd(vma, address, pmd, flags);
@@ -292,6 +300,14 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma,
                        return page;
                return no_page_table(vma, flags);
        }
+       if (is_hugepd(__hugepd(pud_val(*pud)))) {
+               page = follow_huge_pd(vma, address,
+                                     __hugepd(pud_val(*pud)), flags,
+                                     PUD_SHIFT);
+               if (page)
+                       return page;
+               return no_page_table(vma, flags);
+       }
        if (pud_devmap(*pud)) {
                ptl = pud_lock(mm, pud);
                page = follow_devmap_pud(vma, address, pud, flags);
@@ -311,6 +327,7 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma,
                                    unsigned int flags, unsigned int *page_mask)
 {
        p4d_t *p4d;
+       struct page *page;
 
        p4d = p4d_offset(pgdp, address);
        if (p4d_none(*p4d))
@@ -319,6 +336,14 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma,
        if (unlikely(p4d_bad(*p4d)))
                return no_page_table(vma, flags);
 
+       if (is_hugepd(__hugepd(p4d_val(*p4d)))) {
+               page = follow_huge_pd(vma, address,
+                                     __hugepd(p4d_val(*p4d)), flags,
+                                     P4D_SHIFT);
+               if (page)
+                       return page;
+               return no_page_table(vma, flags);
+       }
        return follow_pud_mask(vma, address, p4d, flags, page_mask);
 }
 
@@ -363,6 +388,14 @@ struct page *follow_page_mask(struct vm_area_struct *vma,
                        return page;
                return no_page_table(vma, flags);
        }
+       if (is_hugepd(__hugepd(pgd_val(*pgd)))) {
+               page = follow_huge_pd(vma, address,
+                                     __hugepd(pgd_val(*pgd)), flags,
+                                     PGDIR_SHIFT);
+               if (page)
+                       return page;
+               return no_page_table(vma, flags);
+       }
 
        return follow_p4d_mask(vma, address, pgd, flags, page_mask);
 }
@@ -1390,16 +1423,15 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
                return __gup_device_huge_pmd(orig, addr, end, pages, nr);
 
        refs = 0;
-       head = pmd_page(orig);
-       page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
+       page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
        do {
-               VM_BUG_ON_PAGE(compound_head(page) != head, page);
                pages[*nr] = page;
                (*nr)++;
                page++;
                refs++;
        } while (addr += PAGE_SIZE, addr != end);
 
+       head = compound_head(pmd_page(orig));
        if (!page_cache_add_speculative(head, refs)) {
                *nr -= refs;
                return 0;
@@ -1429,16 +1461,15 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
                return __gup_device_huge_pud(orig, addr, end, pages, nr);
 
        refs = 0;
-       head = pud_page(orig);
-       page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
+       page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
        do {
-               VM_BUG_ON_PAGE(compound_head(page) != head, page);
                pages[*nr] = page;
                (*nr)++;
                page++;
                refs++;
        } while (addr += PAGE_SIZE, addr != end);
 
+       head = compound_head(pud_page(orig));
        if (!page_cache_add_speculative(head, refs)) {
                *nr -= refs;
                return 0;
@@ -1467,16 +1498,15 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
 
        BUILD_BUG_ON(pgd_devmap(orig));
        refs = 0;
-       head = pgd_page(orig);
-       page = head + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
+       page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
        do {
-               VM_BUG_ON_PAGE(compound_head(page) != head, page);
                pages[*nr] = page;
                (*nr)++;
                page++;
                refs++;
        } while (addr += PAGE_SIZE, addr != end);
 
+       head = compound_head(pgd_page(orig));
        if (!page_cache_add_speculative(head, refs)) {
                *nr -= refs;
                return 0;