2 * Generic hugetlb support.
3 * (C) Nadia Yvette Chambers, April 2004
5 #include <linux/list.h>
6 #include <linux/init.h>
8 #include <linux/seq_file.h>
9 #include <linux/sysctl.h>
10 #include <linux/highmem.h>
11 #include <linux/mmu_notifier.h>
12 #include <linux/nodemask.h>
13 #include <linux/pagemap.h>
14 #include <linux/mempolicy.h>
15 #include <linux/compiler.h>
16 #include <linux/cpuset.h>
17 #include <linux/mutex.h>
18 #include <linux/bootmem.h>
19 #include <linux/sysfs.h>
20 #include <linux/slab.h>
21 #include <linux/sched/signal.h>
22 #include <linux/rmap.h>
23 #include <linux/swap.h>
24 #include <linux/swapops.h>
25 #include <linux/jhash.h>
28 #include <asm/pgtable.h>
32 #include <linux/hugetlb.h>
33 #include <linux/hugetlb_cgroup.h>
34 #include <linux/node.h>
35 #include <linux/userfaultfd_k.h>
38 int hugepages_treat_as_movable;
40 int hugetlb_max_hstate __read_mostly;
41 unsigned int default_hstate_idx;
42 struct hstate hstates[HUGE_MAX_HSTATE];
44 * Minimum page order among possible hugepage sizes, set to a proper value
47 static unsigned int minimum_order __read_mostly = UINT_MAX;
49 __initdata LIST_HEAD(huge_boot_pages);
51 /* for command line parsing */
52 static struct hstate * __initdata parsed_hstate;
53 static unsigned long __initdata default_hstate_max_huge_pages;
54 static unsigned long __initdata default_hstate_size;
55 static bool __initdata parsed_valid_hugepagesz = true;
58 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
59 * free_huge_pages, and surplus_huge_pages.
61 DEFINE_SPINLOCK(hugetlb_lock);
64 * Serializes faults on the same logical page. This is used to
65 * prevent spurious OOMs when the hugepage pool is fully utilized.
67 static int num_fault_mutexes;
68 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
70 /* Forward declaration */
71 static int hugetlb_acct_memory(struct hstate *h, long delta);
73 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
75 bool free = (spool->count == 0) && (spool->used_hpages == 0);
77 spin_unlock(&spool->lock);
79 /* If no pages are used, and no other handles to the subpool
80 * remain, give up any reservations mased on minimum size and
83 if (spool->min_hpages != -1)
84 hugetlb_acct_memory(spool->hstate,
90 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
93 struct hugepage_subpool *spool;
95 spool = kzalloc(sizeof(*spool), GFP_KERNEL);
99 spin_lock_init(&spool->lock);
101 spool->max_hpages = max_hpages;
103 spool->min_hpages = min_hpages;
105 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
109 spool->rsv_hpages = min_hpages;
114 void hugepage_put_subpool(struct hugepage_subpool *spool)
116 spin_lock(&spool->lock);
117 BUG_ON(!spool->count);
119 unlock_or_release_subpool(spool);
123 * Subpool accounting for allocating and reserving pages.
124 * Return -ENOMEM if there are not enough resources to satisfy the
125 * the request. Otherwise, return the number of pages by which the
126 * global pools must be adjusted (upward). The returned value may
127 * only be different than the passed value (delta) in the case where
128 * a subpool minimum size must be manitained.
130 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
138 spin_lock(&spool->lock);
140 if (spool->max_hpages != -1) { /* maximum size accounting */
141 if ((spool->used_hpages + delta) <= spool->max_hpages)
142 spool->used_hpages += delta;
149 /* minimum size accounting */
150 if (spool->min_hpages != -1 && spool->rsv_hpages) {
151 if (delta > spool->rsv_hpages) {
153 * Asking for more reserves than those already taken on
154 * behalf of subpool. Return difference.
156 ret = delta - spool->rsv_hpages;
157 spool->rsv_hpages = 0;
159 ret = 0; /* reserves already accounted for */
160 spool->rsv_hpages -= delta;
165 spin_unlock(&spool->lock);
170 * Subpool accounting for freeing and unreserving pages.
171 * Return the number of global page reservations that must be dropped.
172 * The return value may only be different than the passed value (delta)
173 * in the case where a subpool minimum size must be maintained.
175 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
183 spin_lock(&spool->lock);
185 if (spool->max_hpages != -1) /* maximum size accounting */
186 spool->used_hpages -= delta;
188 /* minimum size accounting */
189 if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
190 if (spool->rsv_hpages + delta <= spool->min_hpages)
193 ret = spool->rsv_hpages + delta - spool->min_hpages;
195 spool->rsv_hpages += delta;
196 if (spool->rsv_hpages > spool->min_hpages)
197 spool->rsv_hpages = spool->min_hpages;
201 * If hugetlbfs_put_super couldn't free spool due to an outstanding
202 * quota reference, free it now.
204 unlock_or_release_subpool(spool);
209 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
211 return HUGETLBFS_SB(inode->i_sb)->spool;
214 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
216 return subpool_inode(file_inode(vma->vm_file));
220 * Region tracking -- allows tracking of reservations and instantiated pages
221 * across the pages in a mapping.
223 * The region data structures are embedded into a resv_map and protected
224 * by a resv_map's lock. The set of regions within the resv_map represent
225 * reservations for huge pages, or huge pages that have already been
226 * instantiated within the map. The from and to elements are huge page
227 * indicies into the associated mapping. from indicates the starting index
228 * of the region. to represents the first index past the end of the region.
230 * For example, a file region structure with from == 0 and to == 4 represents
231 * four huge pages in a mapping. It is important to note that the to element
232 * represents the first element past the end of the region. This is used in
233 * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
235 * Interval notation of the form [from, to) will be used to indicate that
236 * the endpoint from is inclusive and to is exclusive.
239 struct list_head link;
245 * Add the huge page range represented by [f, t) to the reserve
246 * map. In the normal case, existing regions will be expanded
247 * to accommodate the specified range. Sufficient regions should
248 * exist for expansion due to the previous call to region_chg
249 * with the same range. However, it is possible that region_del
250 * could have been called after region_chg and modifed the map
251 * in such a way that no region exists to be expanded. In this
252 * case, pull a region descriptor from the cache associated with
253 * the map and use that for the new range.
255 * Return the number of new huge pages added to the map. This
256 * number is greater than or equal to zero.
258 static long region_add(struct resv_map *resv, long f, long t)
260 struct list_head *head = &resv->regions;
261 struct file_region *rg, *nrg, *trg;
264 spin_lock(&resv->lock);
265 /* Locate the region we are either in or before. */
266 list_for_each_entry(rg, head, link)
271 * If no region exists which can be expanded to include the
272 * specified range, the list must have been modified by an
273 * interleving call to region_del(). Pull a region descriptor
274 * from the cache and use it for this range.
276 if (&rg->link == head || t < rg->from) {
277 VM_BUG_ON(resv->region_cache_count <= 0);
279 resv->region_cache_count--;
280 nrg = list_first_entry(&resv->region_cache, struct file_region,
282 list_del(&nrg->link);
286 list_add(&nrg->link, rg->link.prev);
292 /* Round our left edge to the current segment if it encloses us. */
296 /* Check for and consume any regions we now overlap with. */
298 list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
299 if (&rg->link == head)
304 /* If this area reaches higher then extend our area to
305 * include it completely. If this is not the first area
306 * which we intend to reuse, free it. */
310 /* Decrement return value by the deleted range.
311 * Another range will span this area so that by
312 * end of routine add will be >= zero
314 add -= (rg->to - rg->from);
320 add += (nrg->from - f); /* Added to beginning of region */
322 add += t - nrg->to; /* Added to end of region */
326 resv->adds_in_progress--;
327 spin_unlock(&resv->lock);
333 * Examine the existing reserve map and determine how many
334 * huge pages in the specified range [f, t) are NOT currently
335 * represented. This routine is called before a subsequent
336 * call to region_add that will actually modify the reserve
337 * map to add the specified range [f, t). region_chg does
338 * not change the number of huge pages represented by the
339 * map. However, if the existing regions in the map can not
340 * be expanded to represent the new range, a new file_region
341 * structure is added to the map as a placeholder. This is
342 * so that the subsequent region_add call will have all the
343 * regions it needs and will not fail.
345 * Upon entry, region_chg will also examine the cache of region descriptors
346 * associated with the map. If there are not enough descriptors cached, one
347 * will be allocated for the in progress add operation.
349 * Returns the number of huge pages that need to be added to the existing
350 * reservation map for the range [f, t). This number is greater or equal to
351 * zero. -ENOMEM is returned if a new file_region structure or cache entry
352 * is needed and can not be allocated.
354 static long region_chg(struct resv_map *resv, long f, long t)
356 struct list_head *head = &resv->regions;
357 struct file_region *rg, *nrg = NULL;
361 spin_lock(&resv->lock);
363 resv->adds_in_progress++;
366 * Check for sufficient descriptors in the cache to accommodate
367 * the number of in progress add operations.
369 if (resv->adds_in_progress > resv->region_cache_count) {
370 struct file_region *trg;
372 VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1);
373 /* Must drop lock to allocate a new descriptor. */
374 resv->adds_in_progress--;
375 spin_unlock(&resv->lock);
377 trg = kmalloc(sizeof(*trg), GFP_KERNEL);
383 spin_lock(&resv->lock);
384 list_add(&trg->link, &resv->region_cache);
385 resv->region_cache_count++;
389 /* Locate the region we are before or in. */
390 list_for_each_entry(rg, head, link)
394 /* If we are below the current region then a new region is required.
395 * Subtle, allocate a new region at the position but make it zero
396 * size such that we can guarantee to record the reservation. */
397 if (&rg->link == head || t < rg->from) {
399 resv->adds_in_progress--;
400 spin_unlock(&resv->lock);
401 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
407 INIT_LIST_HEAD(&nrg->link);
411 list_add(&nrg->link, rg->link.prev);
416 /* Round our left edge to the current segment if it encloses us. */
421 /* Check for and consume any regions we now overlap with. */
422 list_for_each_entry(rg, rg->link.prev, link) {
423 if (&rg->link == head)
428 /* We overlap with this area, if it extends further than
429 * us then we must extend ourselves. Account for its
430 * existing reservation. */
435 chg -= rg->to - rg->from;
439 spin_unlock(&resv->lock);
440 /* We already know we raced and no longer need the new region */
444 spin_unlock(&resv->lock);
449 * Abort the in progress add operation. The adds_in_progress field
450 * of the resv_map keeps track of the operations in progress between
451 * calls to region_chg and region_add. Operations are sometimes
452 * aborted after the call to region_chg. In such cases, region_abort
453 * is called to decrement the adds_in_progress counter.
455 * NOTE: The range arguments [f, t) are not needed or used in this
456 * routine. They are kept to make reading the calling code easier as
457 * arguments will match the associated region_chg call.
459 static void region_abort(struct resv_map *resv, long f, long t)
461 spin_lock(&resv->lock);
462 VM_BUG_ON(!resv->region_cache_count);
463 resv->adds_in_progress--;
464 spin_unlock(&resv->lock);
468 * Delete the specified range [f, t) from the reserve map. If the
469 * t parameter is LONG_MAX, this indicates that ALL regions after f
470 * should be deleted. Locate the regions which intersect [f, t)
471 * and either trim, delete or split the existing regions.
473 * Returns the number of huge pages deleted from the reserve map.
474 * In the normal case, the return value is zero or more. In the
475 * case where a region must be split, a new region descriptor must
476 * be allocated. If the allocation fails, -ENOMEM will be returned.
477 * NOTE: If the parameter t == LONG_MAX, then we will never split
478 * a region and possibly return -ENOMEM. Callers specifying
479 * t == LONG_MAX do not need to check for -ENOMEM error.
481 static long region_del(struct resv_map *resv, long f, long t)
483 struct list_head *head = &resv->regions;
484 struct file_region *rg, *trg;
485 struct file_region *nrg = NULL;
489 spin_lock(&resv->lock);
490 list_for_each_entry_safe(rg, trg, head, link) {
492 * Skip regions before the range to be deleted. file_region
493 * ranges are normally of the form [from, to). However, there
494 * may be a "placeholder" entry in the map which is of the form
495 * (from, to) with from == to. Check for placeholder entries
496 * at the beginning of the range to be deleted.
498 if (rg->to <= f && (rg->to != rg->from || rg->to != f))
504 if (f > rg->from && t < rg->to) { /* Must split region */
506 * Check for an entry in the cache before dropping
507 * lock and attempting allocation.
510 resv->region_cache_count > resv->adds_in_progress) {
511 nrg = list_first_entry(&resv->region_cache,
514 list_del(&nrg->link);
515 resv->region_cache_count--;
519 spin_unlock(&resv->lock);
520 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
528 /* New entry for end of split region */
531 INIT_LIST_HEAD(&nrg->link);
533 /* Original entry is trimmed */
536 list_add(&nrg->link, &rg->link);
541 if (f <= rg->from && t >= rg->to) { /* Remove entire region */
542 del += rg->to - rg->from;
548 if (f <= rg->from) { /* Trim beginning of region */
551 } else { /* Trim end of region */
557 spin_unlock(&resv->lock);
563 * A rare out of memory error was encountered which prevented removal of
564 * the reserve map region for a page. The huge page itself was free'ed
565 * and removed from the page cache. This routine will adjust the subpool
566 * usage count, and the global reserve count if needed. By incrementing
567 * these counts, the reserve map entry which could not be deleted will
568 * appear as a "reserved" entry instead of simply dangling with incorrect
571 void hugetlb_fix_reserve_counts(struct inode *inode)
573 struct hugepage_subpool *spool = subpool_inode(inode);
576 rsv_adjust = hugepage_subpool_get_pages(spool, 1);
578 struct hstate *h = hstate_inode(inode);
580 hugetlb_acct_memory(h, 1);
585 * Count and return the number of huge pages in the reserve map
586 * that intersect with the range [f, t).
588 static long region_count(struct resv_map *resv, long f, long t)
590 struct list_head *head = &resv->regions;
591 struct file_region *rg;
594 spin_lock(&resv->lock);
595 /* Locate each segment we overlap with, and count that overlap. */
596 list_for_each_entry(rg, head, link) {
605 seg_from = max(rg->from, f);
606 seg_to = min(rg->to, t);
608 chg += seg_to - seg_from;
610 spin_unlock(&resv->lock);
616 * Convert the address within this vma to the page offset within
617 * the mapping, in pagecache page units; huge pages here.
619 static pgoff_t vma_hugecache_offset(struct hstate *h,
620 struct vm_area_struct *vma, unsigned long address)
622 return ((address - vma->vm_start) >> huge_page_shift(h)) +
623 (vma->vm_pgoff >> huge_page_order(h));
626 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
627 unsigned long address)
629 return vma_hugecache_offset(hstate_vma(vma), vma, address);
631 EXPORT_SYMBOL_GPL(linear_hugepage_index);
634 * Return the size of the pages allocated when backing a VMA. In the majority
635 * cases this will be same size as used by the page table entries.
637 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
639 struct hstate *hstate;
641 if (!is_vm_hugetlb_page(vma))
644 hstate = hstate_vma(vma);
646 return 1UL << huge_page_shift(hstate);
648 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
651 * Return the page size being used by the MMU to back a VMA. In the majority
652 * of cases, the page size used by the kernel matches the MMU size. On
653 * architectures where it differs, an architecture-specific version of this
654 * function is required.
656 #ifndef vma_mmu_pagesize
657 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
659 return vma_kernel_pagesize(vma);
664 * Flags for MAP_PRIVATE reservations. These are stored in the bottom
665 * bits of the reservation map pointer, which are always clear due to
668 #define HPAGE_RESV_OWNER (1UL << 0)
669 #define HPAGE_RESV_UNMAPPED (1UL << 1)
670 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
673 * These helpers are used to track how many pages are reserved for
674 * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
675 * is guaranteed to have their future faults succeed.
677 * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
678 * the reserve counters are updated with the hugetlb_lock held. It is safe
679 * to reset the VMA at fork() time as it is not in use yet and there is no
680 * chance of the global counters getting corrupted as a result of the values.
682 * The private mapping reservation is represented in a subtly different
683 * manner to a shared mapping. A shared mapping has a region map associated
684 * with the underlying file, this region map represents the backing file
685 * pages which have ever had a reservation assigned which this persists even
686 * after the page is instantiated. A private mapping has a region map
687 * associated with the original mmap which is attached to all VMAs which
688 * reference it, this region map represents those offsets which have consumed
689 * reservation ie. where pages have been instantiated.
691 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
693 return (unsigned long)vma->vm_private_data;
696 static void set_vma_private_data(struct vm_area_struct *vma,
699 vma->vm_private_data = (void *)value;
702 struct resv_map *resv_map_alloc(void)
704 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
705 struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
707 if (!resv_map || !rg) {
713 kref_init(&resv_map->refs);
714 spin_lock_init(&resv_map->lock);
715 INIT_LIST_HEAD(&resv_map->regions);
717 resv_map->adds_in_progress = 0;
719 INIT_LIST_HEAD(&resv_map->region_cache);
720 list_add(&rg->link, &resv_map->region_cache);
721 resv_map->region_cache_count = 1;
726 void resv_map_release(struct kref *ref)
728 struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
729 struct list_head *head = &resv_map->region_cache;
730 struct file_region *rg, *trg;
732 /* Clear out any active regions before we release the map. */
733 region_del(resv_map, 0, LONG_MAX);
735 /* ... and any entries left in the cache */
736 list_for_each_entry_safe(rg, trg, head, link) {
741 VM_BUG_ON(resv_map->adds_in_progress);
746 static inline struct resv_map *inode_resv_map(struct inode *inode)
748 return inode->i_mapping->private_data;
751 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
753 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
754 if (vma->vm_flags & VM_MAYSHARE) {
755 struct address_space *mapping = vma->vm_file->f_mapping;
756 struct inode *inode = mapping->host;
758 return inode_resv_map(inode);
761 return (struct resv_map *)(get_vma_private_data(vma) &
766 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
768 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
769 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
771 set_vma_private_data(vma, (get_vma_private_data(vma) &
772 HPAGE_RESV_MASK) | (unsigned long)map);
775 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
777 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
778 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
780 set_vma_private_data(vma, get_vma_private_data(vma) | flags);
783 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
785 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
787 return (get_vma_private_data(vma) & flag) != 0;
790 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
791 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
793 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
794 if (!(vma->vm_flags & VM_MAYSHARE))
795 vma->vm_private_data = (void *)0;
798 /* Returns true if the VMA has associated reserve pages */
799 static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
801 if (vma->vm_flags & VM_NORESERVE) {
803 * This address is already reserved by other process(chg == 0),
804 * so, we should decrement reserved count. Without decrementing,
805 * reserve count remains after releasing inode, because this
806 * allocated page will go into page cache and is regarded as
807 * coming from reserved pool in releasing step. Currently, we
808 * don't have any other solution to deal with this situation
809 * properly, so add work-around here.
811 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
817 /* Shared mappings always use reserves */
818 if (vma->vm_flags & VM_MAYSHARE) {
820 * We know VM_NORESERVE is not set. Therefore, there SHOULD
821 * be a region map for all pages. The only situation where
822 * there is no region map is if a hole was punched via
823 * fallocate. In this case, there really are no reverves to
824 * use. This situation is indicated if chg != 0.
833 * Only the process that called mmap() has reserves for
836 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
838 * Like the shared case above, a hole punch or truncate
839 * could have been performed on the private mapping.
840 * Examine the value of chg to determine if reserves
841 * actually exist or were previously consumed.
842 * Very Subtle - The value of chg comes from a previous
843 * call to vma_needs_reserves(). The reserve map for
844 * private mappings has different (opposite) semantics
845 * than that of shared mappings. vma_needs_reserves()
846 * has already taken this difference in semantics into
847 * account. Therefore, the meaning of chg is the same
848 * as in the shared case above. Code could easily be
849 * combined, but keeping it separate draws attention to
850 * subtle differences.
861 static void enqueue_huge_page(struct hstate *h, struct page *page)
863 int nid = page_to_nid(page);
864 list_move(&page->lru, &h->hugepage_freelists[nid]);
865 h->free_huge_pages++;
866 h->free_huge_pages_node[nid]++;
869 static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid)
873 list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
874 if (!PageHWPoison(page))
877 * if 'non-isolated free hugepage' not found on the list,
878 * the allocation fails.
880 if (&h->hugepage_freelists[nid] == &page->lru)
882 list_move(&page->lru, &h->hugepage_activelist);
883 set_page_refcounted(page);
884 h->free_huge_pages--;
885 h->free_huge_pages_node[nid]--;
889 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
894 if (nid != NUMA_NO_NODE)
895 return dequeue_huge_page_node_exact(h, nid);
897 for_each_online_node(node) {
898 page = dequeue_huge_page_node_exact(h, node);
905 /* Movability of hugepages depends on migration support. */
906 static inline gfp_t htlb_alloc_mask(struct hstate *h)
908 if (hugepages_treat_as_movable || hugepage_migration_supported(h))
909 return GFP_HIGHUSER_MOVABLE;
914 static struct page *dequeue_huge_page_vma(struct hstate *h,
915 struct vm_area_struct *vma,
916 unsigned long address, int avoid_reserve,
919 struct page *page = NULL;
920 struct mempolicy *mpol;
921 nodemask_t *nodemask;
924 struct zonelist *zonelist;
927 unsigned int cpuset_mems_cookie;
930 * A child process with MAP_PRIVATE mappings created by their parent
931 * have no page reserves. This check ensures that reservations are
932 * not "stolen". The child may still get SIGKILLed
934 if (!vma_has_reserves(vma, chg) &&
935 h->free_huge_pages - h->resv_huge_pages == 0)
938 /* If reserves cannot be used, ensure enough pages are in the pool */
939 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
943 cpuset_mems_cookie = read_mems_allowed_begin();
944 gfp_mask = htlb_alloc_mask(h);
945 nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
946 zonelist = node_zonelist(nid, gfp_mask);
948 for_each_zone_zonelist_nodemask(zone, z, zonelist,
949 MAX_NR_ZONES - 1, nodemask) {
950 if (cpuset_zone_allowed(zone, gfp_mask)) {
951 page = dequeue_huge_page_node(h, zone_to_nid(zone));
955 if (!vma_has_reserves(vma, chg))
958 SetPagePrivate(page);
959 h->resv_huge_pages--;
966 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
975 * common helper functions for hstate_next_node_to_{alloc|free}.
976 * We may have allocated or freed a huge page based on a different
977 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
978 * be outside of *nodes_allowed. Ensure that we use an allowed
979 * node for alloc or free.
981 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
983 nid = next_node_in(nid, *nodes_allowed);
984 VM_BUG_ON(nid >= MAX_NUMNODES);
989 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
991 if (!node_isset(nid, *nodes_allowed))
992 nid = next_node_allowed(nid, nodes_allowed);
997 * returns the previously saved node ["this node"] from which to
998 * allocate a persistent huge page for the pool and advance the
999 * next node from which to allocate, handling wrap at end of node
1002 static int hstate_next_node_to_alloc(struct hstate *h,
1003 nodemask_t *nodes_allowed)
1007 VM_BUG_ON(!nodes_allowed);
1009 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
1010 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
1016 * helper for free_pool_huge_page() - return the previously saved
1017 * node ["this node"] from which to free a huge page. Advance the
1018 * next node id whether or not we find a free huge page to free so
1019 * that the next attempt to free addresses the next node.
1021 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1025 VM_BUG_ON(!nodes_allowed);
1027 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1028 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1033 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \
1034 for (nr_nodes = nodes_weight(*mask); \
1036 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \
1039 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \
1040 for (nr_nodes = nodes_weight(*mask); \
1042 ((node = hstate_next_node_to_free(hs, mask)) || 1); \
1045 #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
1046 static void destroy_compound_gigantic_page(struct page *page,
1050 int nr_pages = 1 << order;
1051 struct page *p = page + 1;
1053 atomic_set(compound_mapcount_ptr(page), 0);
1054 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1055 clear_compound_head(p);
1056 set_page_refcounted(p);
1059 set_compound_order(page, 0);
1060 __ClearPageHead(page);
1063 static void free_gigantic_page(struct page *page, unsigned int order)
1065 free_contig_range(page_to_pfn(page), 1 << order);
1068 static int __alloc_gigantic_page(unsigned long start_pfn,
1069 unsigned long nr_pages)
1071 unsigned long end_pfn = start_pfn + nr_pages;
1072 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE,
1076 static bool pfn_range_valid_gigantic(struct zone *z,
1077 unsigned long start_pfn, unsigned long nr_pages)
1079 unsigned long i, end_pfn = start_pfn + nr_pages;
1082 for (i = start_pfn; i < end_pfn; i++) {
1086 page = pfn_to_page(i);
1088 if (page_zone(page) != z)
1091 if (PageReserved(page))
1094 if (page_count(page) > 0)
1104 static bool zone_spans_last_pfn(const struct zone *zone,
1105 unsigned long start_pfn, unsigned long nr_pages)
1107 unsigned long last_pfn = start_pfn + nr_pages - 1;
1108 return zone_spans_pfn(zone, last_pfn);
1111 static struct page *alloc_gigantic_page(int nid, unsigned int order)
1113 unsigned long nr_pages = 1 << order;
1114 unsigned long ret, pfn, flags;
1117 z = NODE_DATA(nid)->node_zones;
1118 for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
1119 spin_lock_irqsave(&z->lock, flags);
1121 pfn = ALIGN(z->zone_start_pfn, nr_pages);
1122 while (zone_spans_last_pfn(z, pfn, nr_pages)) {
1123 if (pfn_range_valid_gigantic(z, pfn, nr_pages)) {
1125 * We release the zone lock here because
1126 * alloc_contig_range() will also lock the zone
1127 * at some point. If there's an allocation
1128 * spinning on this lock, it may win the race
1129 * and cause alloc_contig_range() to fail...
1131 spin_unlock_irqrestore(&z->lock, flags);
1132 ret = __alloc_gigantic_page(pfn, nr_pages);
1134 return pfn_to_page(pfn);
1135 spin_lock_irqsave(&z->lock, flags);
1140 spin_unlock_irqrestore(&z->lock, flags);
1146 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
1147 static void prep_compound_gigantic_page(struct page *page, unsigned int order);
1149 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
1153 page = alloc_gigantic_page(nid, huge_page_order(h));
1155 prep_compound_gigantic_page(page, huge_page_order(h));
1156 prep_new_huge_page(h, page, nid);
1162 static int alloc_fresh_gigantic_page(struct hstate *h,
1163 nodemask_t *nodes_allowed)
1165 struct page *page = NULL;
1168 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1169 page = alloc_fresh_gigantic_page_node(h, node);
1177 #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */
1178 static inline bool gigantic_page_supported(void) { return false; }
1179 static inline void free_gigantic_page(struct page *page, unsigned int order) { }
1180 static inline void destroy_compound_gigantic_page(struct page *page,
1181 unsigned int order) { }
1182 static inline int alloc_fresh_gigantic_page(struct hstate *h,
1183 nodemask_t *nodes_allowed) { return 0; }
1186 static void update_and_free_page(struct hstate *h, struct page *page)
1190 if (hstate_is_gigantic(h) && !gigantic_page_supported())
1194 h->nr_huge_pages_node[page_to_nid(page)]--;
1195 for (i = 0; i < pages_per_huge_page(h); i++) {
1196 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
1197 1 << PG_referenced | 1 << PG_dirty |
1198 1 << PG_active | 1 << PG_private |
1201 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
1202 set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
1203 set_page_refcounted(page);
1204 if (hstate_is_gigantic(h)) {
1205 destroy_compound_gigantic_page(page, huge_page_order(h));
1206 free_gigantic_page(page, huge_page_order(h));
1208 __free_pages(page, huge_page_order(h));
1212 struct hstate *size_to_hstate(unsigned long size)
1216 for_each_hstate(h) {
1217 if (huge_page_size(h) == size)
1224 * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
1225 * to hstate->hugepage_activelist.)
1227 * This function can be called for tail pages, but never returns true for them.
1229 bool page_huge_active(struct page *page)
1231 VM_BUG_ON_PAGE(!PageHuge(page), page);
1232 return PageHead(page) && PagePrivate(&page[1]);
1235 /* never called for tail page */
1236 static void set_page_huge_active(struct page *page)
1238 VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1239 SetPagePrivate(&page[1]);
1242 static void clear_page_huge_active(struct page *page)
1244 VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1245 ClearPagePrivate(&page[1]);
1248 void free_huge_page(struct page *page)
1251 * Can't pass hstate in here because it is called from the
1252 * compound page destructor.
1254 struct hstate *h = page_hstate(page);
1255 int nid = page_to_nid(page);
1256 struct hugepage_subpool *spool =
1257 (struct hugepage_subpool *)page_private(page);
1258 bool restore_reserve;
1260 set_page_private(page, 0);
1261 page->mapping = NULL;
1262 VM_BUG_ON_PAGE(page_count(page), page);
1263 VM_BUG_ON_PAGE(page_mapcount(page), page);
1264 restore_reserve = PagePrivate(page);
1265 ClearPagePrivate(page);
1268 * A return code of zero implies that the subpool will be under its
1269 * minimum size if the reservation is not restored after page is free.
1270 * Therefore, force restore_reserve operation.
1272 if (hugepage_subpool_put_pages(spool, 1) == 0)
1273 restore_reserve = true;
1275 spin_lock(&hugetlb_lock);
1276 clear_page_huge_active(page);
1277 hugetlb_cgroup_uncharge_page(hstate_index(h),
1278 pages_per_huge_page(h), page);
1279 if (restore_reserve)
1280 h->resv_huge_pages++;
1282 if (h->surplus_huge_pages_node[nid]) {
1283 /* remove the page from active list */
1284 list_del(&page->lru);
1285 update_and_free_page(h, page);
1286 h->surplus_huge_pages--;
1287 h->surplus_huge_pages_node[nid]--;
1289 arch_clear_hugepage_flags(page);
1290 enqueue_huge_page(h, page);
1292 spin_unlock(&hugetlb_lock);
1295 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1297 INIT_LIST_HEAD(&page->lru);
1298 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1299 spin_lock(&hugetlb_lock);
1300 set_hugetlb_cgroup(page, NULL);
1302 h->nr_huge_pages_node[nid]++;
1303 spin_unlock(&hugetlb_lock);
1304 put_page(page); /* free it into the hugepage allocator */
1307 static void prep_compound_gigantic_page(struct page *page, unsigned int order)
1310 int nr_pages = 1 << order;
1311 struct page *p = page + 1;
1313 /* we rely on prep_new_huge_page to set the destructor */
1314 set_compound_order(page, order);
1315 __ClearPageReserved(page);
1316 __SetPageHead(page);
1317 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1319 * For gigantic hugepages allocated through bootmem at
1320 * boot, it's safer to be consistent with the not-gigantic
1321 * hugepages and clear the PG_reserved bit from all tail pages
1322 * too. Otherwse drivers using get_user_pages() to access tail
1323 * pages may get the reference counting wrong if they see
1324 * PG_reserved set on a tail page (despite the head page not
1325 * having PG_reserved set). Enforcing this consistency between
1326 * head and tail pages allows drivers to optimize away a check
1327 * on the head page when they need know if put_page() is needed
1328 * after get_user_pages().
1330 __ClearPageReserved(p);
1331 set_page_count(p, 0);
1332 set_compound_head(p, page);
1334 atomic_set(compound_mapcount_ptr(page), -1);
1338 * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1339 * transparent huge pages. See the PageTransHuge() documentation for more
1342 int PageHuge(struct page *page)
1344 if (!PageCompound(page))
1347 page = compound_head(page);
1348 return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
1350 EXPORT_SYMBOL_GPL(PageHuge);
1353 * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1354 * normal or transparent huge pages.
1356 int PageHeadHuge(struct page *page_head)
1358 if (!PageHead(page_head))
1361 return get_compound_page_dtor(page_head) == free_huge_page;
1364 pgoff_t __basepage_index(struct page *page)
1366 struct page *page_head = compound_head(page);
1367 pgoff_t index = page_index(page_head);
1368 unsigned long compound_idx;
1370 if (!PageHuge(page_head))
1371 return page_index(page);
1373 if (compound_order(page_head) >= MAX_ORDER)
1374 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1376 compound_idx = page - page_head;
1378 return (index << compound_order(page_head)) + compound_idx;
1381 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
1385 page = __alloc_pages_node(nid,
1386 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1387 __GFP_REPEAT|__GFP_NOWARN,
1388 huge_page_order(h));
1390 prep_new_huge_page(h, page, nid);
1396 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
1402 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1403 page = alloc_fresh_huge_page_node(h, node);
1411 count_vm_event(HTLB_BUDDY_PGALLOC);
1413 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1419 * Free huge page from pool from next node to free.
1420 * Attempt to keep persistent huge pages more or less
1421 * balanced over allowed nodes.
1422 * Called with hugetlb_lock locked.
1424 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1430 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1432 * If we're returning unused surplus pages, only examine
1433 * nodes with surplus pages.
1435 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1436 !list_empty(&h->hugepage_freelists[node])) {
1438 list_entry(h->hugepage_freelists[node].next,
1440 list_del(&page->lru);
1441 h->free_huge_pages--;
1442 h->free_huge_pages_node[node]--;
1444 h->surplus_huge_pages--;
1445 h->surplus_huge_pages_node[node]--;
1447 update_and_free_page(h, page);
1457 * Dissolve a given free hugepage into free buddy pages. This function does
1458 * nothing for in-use (including surplus) hugepages. Returns -EBUSY if the
1459 * number of free hugepages would be reduced below the number of reserved
1462 int dissolve_free_huge_page(struct page *page)
1466 spin_lock(&hugetlb_lock);
1467 if (PageHuge(page) && !page_count(page)) {
1468 struct page *head = compound_head(page);
1469 struct hstate *h = page_hstate(head);
1470 int nid = page_to_nid(head);
1471 if (h->free_huge_pages - h->resv_huge_pages == 0) {
1476 * Move PageHWPoison flag from head page to the raw error page,
1477 * which makes any subpages rather than the error page reusable.
1479 if (PageHWPoison(head) && page != head) {
1480 SetPageHWPoison(page);
1481 ClearPageHWPoison(head);
1483 list_del(&head->lru);
1484 h->free_huge_pages--;
1485 h->free_huge_pages_node[nid]--;
1486 h->max_huge_pages--;
1487 update_and_free_page(h, head);
1490 spin_unlock(&hugetlb_lock);
1495 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1496 * make specified memory blocks removable from the system.
1497 * Note that this will dissolve a free gigantic hugepage completely, if any
1498 * part of it lies within the given range.
1499 * Also note that if dissolve_free_huge_page() returns with an error, all
1500 * free hugepages that were dissolved before that error are lost.
1502 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1508 if (!hugepages_supported())
1511 for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) {
1512 page = pfn_to_page(pfn);
1513 if (PageHuge(page) && !page_count(page)) {
1514 rc = dissolve_free_huge_page(page);
1524 * There are 3 ways this can get called:
1525 * 1. With vma+addr: we use the VMA's memory policy
1526 * 2. With !vma, but nid=NUMA_NO_NODE: We try to allocate a huge
1527 * page from any node, and let the buddy allocator itself figure
1529 * 3. With !vma, but nid!=NUMA_NO_NODE. We allocate a huge page
1530 * strictly from 'nid'
1532 static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h,
1533 struct vm_area_struct *vma, unsigned long addr, int nid)
1535 int order = huge_page_order(h);
1536 gfp_t gfp = htlb_alloc_mask(h)|__GFP_COMP|__GFP_REPEAT|__GFP_NOWARN;
1537 unsigned int cpuset_mems_cookie;
1540 * We need a VMA to get a memory policy. If we do not
1541 * have one, we use the 'nid' argument.
1543 * The mempolicy stuff below has some non-inlined bits
1544 * and calls ->vm_ops. That makes it hard to optimize at
1545 * compile-time, even when NUMA is off and it does
1546 * nothing. This helps the compiler optimize it out.
1548 if (!IS_ENABLED(CONFIG_NUMA) || !vma) {
1550 * If a specific node is requested, make sure to
1551 * get memory from there, but only when a node
1552 * is explicitly specified.
1554 if (nid != NUMA_NO_NODE)
1555 gfp |= __GFP_THISNODE;
1557 * Make sure to call something that can handle
1560 return alloc_pages_node(nid, gfp, order);
1564 * OK, so we have a VMA. Fetch the mempolicy and try to
1565 * allocate a huge page with it. We will only reach this
1566 * when CONFIG_NUMA=y.
1570 struct mempolicy *mpol;
1572 nodemask_t *nodemask;
1574 cpuset_mems_cookie = read_mems_allowed_begin();
1575 nid = huge_node(vma, addr, gfp, &mpol, &nodemask);
1576 mpol_cond_put(mpol);
1577 page = __alloc_pages_nodemask(gfp, order, nid, nodemask);
1580 } while (read_mems_allowed_retry(cpuset_mems_cookie));
1586 * There are two ways to allocate a huge page:
1587 * 1. When you have a VMA and an address (like a fault)
1588 * 2. When you have no VMA (like when setting /proc/.../nr_hugepages)
1590 * 'vma' and 'addr' are only for (1). 'nid' is always NUMA_NO_NODE in
1591 * this case which signifies that the allocation should be done with
1592 * respect for the VMA's memory policy.
1594 * For (2), we ignore 'vma' and 'addr' and use 'nid' exclusively. This
1595 * implies that memory policies will not be taken in to account.
1597 static struct page *__alloc_buddy_huge_page(struct hstate *h,
1598 struct vm_area_struct *vma, unsigned long addr, int nid)
1603 if (hstate_is_gigantic(h))
1607 * Make sure that anyone specifying 'nid' is not also specifying a VMA.
1608 * This makes sure the caller is picking _one_ of the modes with which
1609 * we can call this function, not both.
1611 if (vma || (addr != -1)) {
1612 VM_WARN_ON_ONCE(addr == -1);
1613 VM_WARN_ON_ONCE(nid != NUMA_NO_NODE);
1616 * Assume we will successfully allocate the surplus page to
1617 * prevent racing processes from causing the surplus to exceed
1620 * This however introduces a different race, where a process B
1621 * tries to grow the static hugepage pool while alloc_pages() is
1622 * called by process A. B will only examine the per-node
1623 * counters in determining if surplus huge pages can be
1624 * converted to normal huge pages in adjust_pool_surplus(). A
1625 * won't be able to increment the per-node counter, until the
1626 * lock is dropped by B, but B doesn't drop hugetlb_lock until
1627 * no more huge pages can be converted from surplus to normal
1628 * state (and doesn't try to convert again). Thus, we have a
1629 * case where a surplus huge page exists, the pool is grown, and
1630 * the surplus huge page still exists after, even though it
1631 * should just have been converted to a normal huge page. This
1632 * does not leak memory, though, as the hugepage will be freed
1633 * once it is out of use. It also does not allow the counters to
1634 * go out of whack in adjust_pool_surplus() as we don't modify
1635 * the node values until we've gotten the hugepage and only the
1636 * per-node value is checked there.
1638 spin_lock(&hugetlb_lock);
1639 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1640 spin_unlock(&hugetlb_lock);
1644 h->surplus_huge_pages++;
1646 spin_unlock(&hugetlb_lock);
1648 page = __hugetlb_alloc_buddy_huge_page(h, vma, addr, nid);
1650 spin_lock(&hugetlb_lock);
1652 INIT_LIST_HEAD(&page->lru);
1653 r_nid = page_to_nid(page);
1654 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1655 set_hugetlb_cgroup(page, NULL);
1657 * We incremented the global counters already
1659 h->nr_huge_pages_node[r_nid]++;
1660 h->surplus_huge_pages_node[r_nid]++;
1661 __count_vm_event(HTLB_BUDDY_PGALLOC);
1664 h->surplus_huge_pages--;
1665 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1667 spin_unlock(&hugetlb_lock);
1673 * Allocate a huge page from 'nid'. Note, 'nid' may be
1674 * NUMA_NO_NODE, which means that it may be allocated
1678 struct page *__alloc_buddy_huge_page_no_mpol(struct hstate *h, int nid)
1680 unsigned long addr = -1;
1682 return __alloc_buddy_huge_page(h, NULL, addr, nid);
1686 * Use the VMA's mpolicy to allocate a huge page from the buddy.
1689 struct page *__alloc_buddy_huge_page_with_mpol(struct hstate *h,
1690 struct vm_area_struct *vma, unsigned long addr)
1692 return __alloc_buddy_huge_page(h, vma, addr, NUMA_NO_NODE);
1696 * This allocation function is useful in the context where vma is irrelevant.
1697 * E.g. soft-offlining uses this function because it only cares physical
1698 * address of error page.
1700 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1702 struct page *page = NULL;
1704 spin_lock(&hugetlb_lock);
1705 if (h->free_huge_pages - h->resv_huge_pages > 0)
1706 page = dequeue_huge_page_node(h, nid);
1707 spin_unlock(&hugetlb_lock);
1710 page = __alloc_buddy_huge_page_no_mpol(h, nid);
1716 * Increase the hugetlb pool such that it can accommodate a reservation
1719 static int gather_surplus_pages(struct hstate *h, int delta)
1721 struct list_head surplus_list;
1722 struct page *page, *tmp;
1724 int needed, allocated;
1725 bool alloc_ok = true;
1727 needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1729 h->resv_huge_pages += delta;
1734 INIT_LIST_HEAD(&surplus_list);
1738 spin_unlock(&hugetlb_lock);
1739 for (i = 0; i < needed; i++) {
1740 page = __alloc_buddy_huge_page_no_mpol(h, NUMA_NO_NODE);
1745 list_add(&page->lru, &surplus_list);
1750 * After retaking hugetlb_lock, we need to recalculate 'needed'
1751 * because either resv_huge_pages or free_huge_pages may have changed.
1753 spin_lock(&hugetlb_lock);
1754 needed = (h->resv_huge_pages + delta) -
1755 (h->free_huge_pages + allocated);
1760 * We were not able to allocate enough pages to
1761 * satisfy the entire reservation so we free what
1762 * we've allocated so far.
1767 * The surplus_list now contains _at_least_ the number of extra pages
1768 * needed to accommodate the reservation. Add the appropriate number
1769 * of pages to the hugetlb pool and free the extras back to the buddy
1770 * allocator. Commit the entire reservation here to prevent another
1771 * process from stealing the pages as they are added to the pool but
1772 * before they are reserved.
1774 needed += allocated;
1775 h->resv_huge_pages += delta;
1778 /* Free the needed pages to the hugetlb pool */
1779 list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1783 * This page is now managed by the hugetlb allocator and has
1784 * no users -- drop the buddy allocator's reference.
1786 put_page_testzero(page);
1787 VM_BUG_ON_PAGE(page_count(page), page);
1788 enqueue_huge_page(h, page);
1791 spin_unlock(&hugetlb_lock);
1793 /* Free unnecessary surplus pages to the buddy allocator */
1794 list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1796 spin_lock(&hugetlb_lock);
1802 * This routine has two main purposes:
1803 * 1) Decrement the reservation count (resv_huge_pages) by the value passed
1804 * in unused_resv_pages. This corresponds to the prior adjustments made
1805 * to the associated reservation map.
1806 * 2) Free any unused surplus pages that may have been allocated to satisfy
1807 * the reservation. As many as unused_resv_pages may be freed.
1809 * Called with hugetlb_lock held. However, the lock could be dropped (and
1810 * reacquired) during calls to cond_resched_lock. Whenever dropping the lock,
1811 * we must make sure nobody else can claim pages we are in the process of
1812 * freeing. Do this by ensuring resv_huge_page always is greater than the
1813 * number of huge pages we plan to free when dropping the lock.
1815 static void return_unused_surplus_pages(struct hstate *h,
1816 unsigned long unused_resv_pages)
1818 unsigned long nr_pages;
1820 /* Cannot return gigantic pages currently */
1821 if (hstate_is_gigantic(h))
1825 * Part (or even all) of the reservation could have been backed
1826 * by pre-allocated pages. Only free surplus pages.
1828 nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1831 * We want to release as many surplus pages as possible, spread
1832 * evenly across all nodes with memory. Iterate across these nodes
1833 * until we can no longer free unreserved surplus pages. This occurs
1834 * when the nodes with surplus pages have no free pages.
1835 * free_pool_huge_page() will balance the the freed pages across the
1836 * on-line nodes with memory and will handle the hstate accounting.
1838 * Note that we decrement resv_huge_pages as we free the pages. If
1839 * we drop the lock, resv_huge_pages will still be sufficiently large
1840 * to cover subsequent pages we may free.
1842 while (nr_pages--) {
1843 h->resv_huge_pages--;
1844 unused_resv_pages--;
1845 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1847 cond_resched_lock(&hugetlb_lock);
1851 /* Fully uncommit the reservation */
1852 h->resv_huge_pages -= unused_resv_pages;
1857 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
1858 * are used by the huge page allocation routines to manage reservations.
1860 * vma_needs_reservation is called to determine if the huge page at addr
1861 * within the vma has an associated reservation. If a reservation is
1862 * needed, the value 1 is returned. The caller is then responsible for
1863 * managing the global reservation and subpool usage counts. After
1864 * the huge page has been allocated, vma_commit_reservation is called
1865 * to add the page to the reservation map. If the page allocation fails,
1866 * the reservation must be ended instead of committed. vma_end_reservation
1867 * is called in such cases.
1869 * In the normal case, vma_commit_reservation returns the same value
1870 * as the preceding vma_needs_reservation call. The only time this
1871 * is not the case is if a reserve map was changed between calls. It
1872 * is the responsibility of the caller to notice the difference and
1873 * take appropriate action.
1875 * vma_add_reservation is used in error paths where a reservation must
1876 * be restored when a newly allocated huge page must be freed. It is
1877 * to be called after calling vma_needs_reservation to determine if a
1878 * reservation exists.
1880 enum vma_resv_mode {
1886 static long __vma_reservation_common(struct hstate *h,
1887 struct vm_area_struct *vma, unsigned long addr,
1888 enum vma_resv_mode mode)
1890 struct resv_map *resv;
1894 resv = vma_resv_map(vma);
1898 idx = vma_hugecache_offset(h, vma, addr);
1900 case VMA_NEEDS_RESV:
1901 ret = region_chg(resv, idx, idx + 1);
1903 case VMA_COMMIT_RESV:
1904 ret = region_add(resv, idx, idx + 1);
1907 region_abort(resv, idx, idx + 1);
1911 if (vma->vm_flags & VM_MAYSHARE)
1912 ret = region_add(resv, idx, idx + 1);
1914 region_abort(resv, idx, idx + 1);
1915 ret = region_del(resv, idx, idx + 1);
1922 if (vma->vm_flags & VM_MAYSHARE)
1924 else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) {
1926 * In most cases, reserves always exist for private mappings.
1927 * However, a file associated with mapping could have been
1928 * hole punched or truncated after reserves were consumed.
1929 * As subsequent fault on such a range will not use reserves.
1930 * Subtle - The reserve map for private mappings has the
1931 * opposite meaning than that of shared mappings. If NO
1932 * entry is in the reserve map, it means a reservation exists.
1933 * If an entry exists in the reserve map, it means the
1934 * reservation has already been consumed. As a result, the
1935 * return value of this routine is the opposite of the
1936 * value returned from reserve map manipulation routines above.
1944 return ret < 0 ? ret : 0;
1947 static long vma_needs_reservation(struct hstate *h,
1948 struct vm_area_struct *vma, unsigned long addr)
1950 return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
1953 static long vma_commit_reservation(struct hstate *h,
1954 struct vm_area_struct *vma, unsigned long addr)
1956 return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
1959 static void vma_end_reservation(struct hstate *h,
1960 struct vm_area_struct *vma, unsigned long addr)
1962 (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
1965 static long vma_add_reservation(struct hstate *h,
1966 struct vm_area_struct *vma, unsigned long addr)
1968 return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
1972 * This routine is called to restore a reservation on error paths. In the
1973 * specific error paths, a huge page was allocated (via alloc_huge_page)
1974 * and is about to be freed. If a reservation for the page existed,
1975 * alloc_huge_page would have consumed the reservation and set PagePrivate
1976 * in the newly allocated page. When the page is freed via free_huge_page,
1977 * the global reservation count will be incremented if PagePrivate is set.
1978 * However, free_huge_page can not adjust the reserve map. Adjust the
1979 * reserve map here to be consistent with global reserve count adjustments
1980 * to be made by free_huge_page.
1982 static void restore_reserve_on_error(struct hstate *h,
1983 struct vm_area_struct *vma, unsigned long address,
1986 if (unlikely(PagePrivate(page))) {
1987 long rc = vma_needs_reservation(h, vma, address);
1989 if (unlikely(rc < 0)) {
1991 * Rare out of memory condition in reserve map
1992 * manipulation. Clear PagePrivate so that
1993 * global reserve count will not be incremented
1994 * by free_huge_page. This will make it appear
1995 * as though the reservation for this page was
1996 * consumed. This may prevent the task from
1997 * faulting in the page at a later time. This
1998 * is better than inconsistent global huge page
1999 * accounting of reserve counts.
2001 ClearPagePrivate(page);
2003 rc = vma_add_reservation(h, vma, address);
2004 if (unlikely(rc < 0))
2006 * See above comment about rare out of
2009 ClearPagePrivate(page);
2011 vma_end_reservation(h, vma, address);
2015 struct page *alloc_huge_page(struct vm_area_struct *vma,
2016 unsigned long addr, int avoid_reserve)
2018 struct hugepage_subpool *spool = subpool_vma(vma);
2019 struct hstate *h = hstate_vma(vma);
2021 long map_chg, map_commit;
2024 struct hugetlb_cgroup *h_cg;
2026 idx = hstate_index(h);
2028 * Examine the region/reserve map to determine if the process
2029 * has a reservation for the page to be allocated. A return
2030 * code of zero indicates a reservation exists (no change).
2032 map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
2034 return ERR_PTR(-ENOMEM);
2037 * Processes that did not create the mapping will have no
2038 * reserves as indicated by the region/reserve map. Check
2039 * that the allocation will not exceed the subpool limit.
2040 * Allocations for MAP_NORESERVE mappings also need to be
2041 * checked against any subpool limit.
2043 if (map_chg || avoid_reserve) {
2044 gbl_chg = hugepage_subpool_get_pages(spool, 1);
2046 vma_end_reservation(h, vma, addr);
2047 return ERR_PTR(-ENOSPC);
2051 * Even though there was no reservation in the region/reserve
2052 * map, there could be reservations associated with the
2053 * subpool that can be used. This would be indicated if the
2054 * return value of hugepage_subpool_get_pages() is zero.
2055 * However, if avoid_reserve is specified we still avoid even
2056 * the subpool reservations.
2062 ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
2064 goto out_subpool_put;
2066 spin_lock(&hugetlb_lock);
2068 * glb_chg is passed to indicate whether or not a page must be taken
2069 * from the global free pool (global change). gbl_chg == 0 indicates
2070 * a reservation exists for the allocation.
2072 page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
2074 spin_unlock(&hugetlb_lock);
2075 page = __alloc_buddy_huge_page_with_mpol(h, vma, addr);
2077 goto out_uncharge_cgroup;
2078 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
2079 SetPagePrivate(page);
2080 h->resv_huge_pages--;
2082 spin_lock(&hugetlb_lock);
2083 list_move(&page->lru, &h->hugepage_activelist);
2086 hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
2087 spin_unlock(&hugetlb_lock);
2089 set_page_private(page, (unsigned long)spool);
2091 map_commit = vma_commit_reservation(h, vma, addr);
2092 if (unlikely(map_chg > map_commit)) {
2094 * The page was added to the reservation map between
2095 * vma_needs_reservation and vma_commit_reservation.
2096 * This indicates a race with hugetlb_reserve_pages.
2097 * Adjust for the subpool count incremented above AND
2098 * in hugetlb_reserve_pages for the same page. Also,
2099 * the reservation count added in hugetlb_reserve_pages
2100 * no longer applies.
2104 rsv_adjust = hugepage_subpool_put_pages(spool, 1);
2105 hugetlb_acct_memory(h, -rsv_adjust);
2109 out_uncharge_cgroup:
2110 hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
2112 if (map_chg || avoid_reserve)
2113 hugepage_subpool_put_pages(spool, 1);
2114 vma_end_reservation(h, vma, addr);
2115 return ERR_PTR(-ENOSPC);
2119 * alloc_huge_page()'s wrapper which simply returns the page if allocation
2120 * succeeds, otherwise NULL. This function is called from new_vma_page(),
2121 * where no ERR_VALUE is expected to be returned.
2123 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
2124 unsigned long addr, int avoid_reserve)
2126 struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
2132 int __weak alloc_bootmem_huge_page(struct hstate *h)
2134 struct huge_bootmem_page *m;
2137 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
2140 addr = memblock_virt_alloc_try_nid_nopanic(
2141 huge_page_size(h), huge_page_size(h),
2142 0, BOOTMEM_ALLOC_ACCESSIBLE, node);
2145 * Use the beginning of the huge page to store the
2146 * huge_bootmem_page struct (until gather_bootmem
2147 * puts them into the mem_map).
2156 BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
2157 /* Put them into a private list first because mem_map is not up yet */
2158 list_add(&m->list, &huge_boot_pages);
2163 static void __init prep_compound_huge_page(struct page *page,
2166 if (unlikely(order > (MAX_ORDER - 1)))
2167 prep_compound_gigantic_page(page, order);
2169 prep_compound_page(page, order);
2172 /* Put bootmem huge pages into the standard lists after mem_map is up */
2173 static void __init gather_bootmem_prealloc(void)
2175 struct huge_bootmem_page *m;
2177 list_for_each_entry(m, &huge_boot_pages, list) {
2178 struct hstate *h = m->hstate;
2181 #ifdef CONFIG_HIGHMEM
2182 page = pfn_to_page(m->phys >> PAGE_SHIFT);
2183 memblock_free_late(__pa(m),
2184 sizeof(struct huge_bootmem_page));
2186 page = virt_to_page(m);
2188 WARN_ON(page_count(page) != 1);
2189 prep_compound_huge_page(page, h->order);
2190 WARN_ON(PageReserved(page));
2191 prep_new_huge_page(h, page, page_to_nid(page));
2193 * If we had gigantic hugepages allocated at boot time, we need
2194 * to restore the 'stolen' pages to totalram_pages in order to
2195 * fix confusing memory reports from free(1) and another
2196 * side-effects, like CommitLimit going negative.
2198 if (hstate_is_gigantic(h))
2199 adjust_managed_page_count(page, 1 << h->order);
2203 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
2207 for (i = 0; i < h->max_huge_pages; ++i) {
2208 if (hstate_is_gigantic(h)) {
2209 if (!alloc_bootmem_huge_page(h))
2211 } else if (!alloc_fresh_huge_page(h,
2212 &node_states[N_MEMORY]))
2215 h->max_huge_pages = i;
2218 static void __init hugetlb_init_hstates(void)
2222 for_each_hstate(h) {
2223 if (minimum_order > huge_page_order(h))
2224 minimum_order = huge_page_order(h);
2226 /* oversize hugepages were init'ed in early boot */
2227 if (!hstate_is_gigantic(h))
2228 hugetlb_hstate_alloc_pages(h);
2230 VM_BUG_ON(minimum_order == UINT_MAX);
2233 static char * __init memfmt(char *buf, unsigned long n)
2235 if (n >= (1UL << 30))
2236 sprintf(buf, "%lu GB", n >> 30);
2237 else if (n >= (1UL << 20))
2238 sprintf(buf, "%lu MB", n >> 20);
2240 sprintf(buf, "%lu KB", n >> 10);
2244 static void __init report_hugepages(void)
2248 for_each_hstate(h) {
2250 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
2251 memfmt(buf, huge_page_size(h)),
2252 h->free_huge_pages);
2256 #ifdef CONFIG_HIGHMEM
2257 static void try_to_free_low(struct hstate *h, unsigned long count,
2258 nodemask_t *nodes_allowed)
2262 if (hstate_is_gigantic(h))
2265 for_each_node_mask(i, *nodes_allowed) {
2266 struct page *page, *next;
2267 struct list_head *freel = &h->hugepage_freelists[i];
2268 list_for_each_entry_safe(page, next, freel, lru) {
2269 if (count >= h->nr_huge_pages)
2271 if (PageHighMem(page))
2273 list_del(&page->lru);
2274 update_and_free_page(h, page);
2275 h->free_huge_pages--;
2276 h->free_huge_pages_node[page_to_nid(page)]--;
2281 static inline void try_to_free_low(struct hstate *h, unsigned long count,
2282 nodemask_t *nodes_allowed)
2288 * Increment or decrement surplus_huge_pages. Keep node-specific counters
2289 * balanced by operating on them in a round-robin fashion.
2290 * Returns 1 if an adjustment was made.
2292 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
2297 VM_BUG_ON(delta != -1 && delta != 1);
2300 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2301 if (h->surplus_huge_pages_node[node])
2305 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2306 if (h->surplus_huge_pages_node[node] <
2307 h->nr_huge_pages_node[node])
2314 h->surplus_huge_pages += delta;
2315 h->surplus_huge_pages_node[node] += delta;
2319 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2320 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
2321 nodemask_t *nodes_allowed)
2323 unsigned long min_count, ret;
2325 if (hstate_is_gigantic(h) && !gigantic_page_supported())
2326 return h->max_huge_pages;
2329 * Increase the pool size
2330 * First take pages out of surplus state. Then make up the
2331 * remaining difference by allocating fresh huge pages.
2333 * We might race with __alloc_buddy_huge_page() here and be unable
2334 * to convert a surplus huge page to a normal huge page. That is
2335 * not critical, though, it just means the overall size of the
2336 * pool might be one hugepage larger than it needs to be, but
2337 * within all the constraints specified by the sysctls.
2339 spin_lock(&hugetlb_lock);
2340 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
2341 if (!adjust_pool_surplus(h, nodes_allowed, -1))
2345 while (count > persistent_huge_pages(h)) {
2347 * If this allocation races such that we no longer need the
2348 * page, free_huge_page will handle it by freeing the page
2349 * and reducing the surplus.
2351 spin_unlock(&hugetlb_lock);
2353 /* yield cpu to avoid soft lockup */
2356 if (hstate_is_gigantic(h))
2357 ret = alloc_fresh_gigantic_page(h, nodes_allowed);
2359 ret = alloc_fresh_huge_page(h, nodes_allowed);
2360 spin_lock(&hugetlb_lock);
2364 /* Bail for signals. Probably ctrl-c from user */
2365 if (signal_pending(current))
2370 * Decrease the pool size
2371 * First return free pages to the buddy allocator (being careful
2372 * to keep enough around to satisfy reservations). Then place
2373 * pages into surplus state as needed so the pool will shrink
2374 * to the desired size as pages become free.
2376 * By placing pages into the surplus state independent of the
2377 * overcommit value, we are allowing the surplus pool size to
2378 * exceed overcommit. There are few sane options here. Since
2379 * __alloc_buddy_huge_page() is checking the global counter,
2380 * though, we'll note that we're not allowed to exceed surplus
2381 * and won't grow the pool anywhere else. Not until one of the
2382 * sysctls are changed, or the surplus pages go out of use.
2384 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
2385 min_count = max(count, min_count);
2386 try_to_free_low(h, min_count, nodes_allowed);
2387 while (min_count < persistent_huge_pages(h)) {
2388 if (!free_pool_huge_page(h, nodes_allowed, 0))
2390 cond_resched_lock(&hugetlb_lock);
2392 while (count < persistent_huge_pages(h)) {
2393 if (!adjust_pool_surplus(h, nodes_allowed, 1))
2397 ret = persistent_huge_pages(h);
2398 spin_unlock(&hugetlb_lock);
2402 #define HSTATE_ATTR_RO(_name) \
2403 static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
2405 #define HSTATE_ATTR(_name) \
2406 static struct kobj_attribute _name##_attr = \
2407 __ATTR(_name, 0644, _name##_show, _name##_store)
2409 static struct kobject *hugepages_kobj;
2410 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2412 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
2414 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
2418 for (i = 0; i < HUGE_MAX_HSTATE; i++)
2419 if (hstate_kobjs[i] == kobj) {
2421 *nidp = NUMA_NO_NODE;
2425 return kobj_to_node_hstate(kobj, nidp);
2428 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
2429 struct kobj_attribute *attr, char *buf)
2432 unsigned long nr_huge_pages;
2435 h = kobj_to_hstate(kobj, &nid);
2436 if (nid == NUMA_NO_NODE)
2437 nr_huge_pages = h->nr_huge_pages;
2439 nr_huge_pages = h->nr_huge_pages_node[nid];
2441 return sprintf(buf, "%lu\n", nr_huge_pages);
2444 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
2445 struct hstate *h, int nid,
2446 unsigned long count, size_t len)
2449 NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
2451 if (hstate_is_gigantic(h) && !gigantic_page_supported()) {
2456 if (nid == NUMA_NO_NODE) {
2458 * global hstate attribute
2460 if (!(obey_mempolicy &&
2461 init_nodemask_of_mempolicy(nodes_allowed))) {
2462 NODEMASK_FREE(nodes_allowed);
2463 nodes_allowed = &node_states[N_MEMORY];
2465 } else if (nodes_allowed) {
2467 * per node hstate attribute: adjust count to global,
2468 * but restrict alloc/free to the specified node.
2470 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
2471 init_nodemask_of_node(nodes_allowed, nid);
2473 nodes_allowed = &node_states[N_MEMORY];
2475 h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
2477 if (nodes_allowed != &node_states[N_MEMORY])
2478 NODEMASK_FREE(nodes_allowed);
2482 NODEMASK_FREE(nodes_allowed);
2486 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
2487 struct kobject *kobj, const char *buf,
2491 unsigned long count;
2495 err = kstrtoul(buf, 10, &count);
2499 h = kobj_to_hstate(kobj, &nid);
2500 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
2503 static ssize_t nr_hugepages_show(struct kobject *kobj,
2504 struct kobj_attribute *attr, char *buf)
2506 return nr_hugepages_show_common(kobj, attr, buf);
2509 static ssize_t nr_hugepages_store(struct kobject *kobj,
2510 struct kobj_attribute *attr, const char *buf, size_t len)
2512 return nr_hugepages_store_common(false, kobj, buf, len);
2514 HSTATE_ATTR(nr_hugepages);
2519 * hstate attribute for optionally mempolicy-based constraint on persistent
2520 * huge page alloc/free.
2522 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
2523 struct kobj_attribute *attr, char *buf)
2525 return nr_hugepages_show_common(kobj, attr, buf);
2528 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
2529 struct kobj_attribute *attr, const char *buf, size_t len)
2531 return nr_hugepages_store_common(true, kobj, buf, len);
2533 HSTATE_ATTR(nr_hugepages_mempolicy);
2537 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
2538 struct kobj_attribute *attr, char *buf)
2540 struct hstate *h = kobj_to_hstate(kobj, NULL);
2541 return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
2544 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
2545 struct kobj_attribute *attr, const char *buf, size_t count)
2548 unsigned long input;
2549 struct hstate *h = kobj_to_hstate(kobj, NULL);
2551 if (hstate_is_gigantic(h))
2554 err = kstrtoul(buf, 10, &input);
2558 spin_lock(&hugetlb_lock);
2559 h->nr_overcommit_huge_pages = input;
2560 spin_unlock(&hugetlb_lock);
2564 HSTATE_ATTR(nr_overcommit_hugepages);
2566 static ssize_t free_hugepages_show(struct kobject *kobj,
2567 struct kobj_attribute *attr, char *buf)
2570 unsigned long free_huge_pages;
2573 h = kobj_to_hstate(kobj, &nid);
2574 if (nid == NUMA_NO_NODE)
2575 free_huge_pages = h->free_huge_pages;
2577 free_huge_pages = h->free_huge_pages_node[nid];
2579 return sprintf(buf, "%lu\n", free_huge_pages);
2581 HSTATE_ATTR_RO(free_hugepages);
2583 static ssize_t resv_hugepages_show(struct kobject *kobj,
2584 struct kobj_attribute *attr, char *buf)
2586 struct hstate *h = kobj_to_hstate(kobj, NULL);
2587 return sprintf(buf, "%lu\n", h->resv_huge_pages);
2589 HSTATE_ATTR_RO(resv_hugepages);
2591 static ssize_t surplus_hugepages_show(struct kobject *kobj,
2592 struct kobj_attribute *attr, char *buf)
2595 unsigned long surplus_huge_pages;
2598 h = kobj_to_hstate(kobj, &nid);
2599 if (nid == NUMA_NO_NODE)
2600 surplus_huge_pages = h->surplus_huge_pages;
2602 surplus_huge_pages = h->surplus_huge_pages_node[nid];
2604 return sprintf(buf, "%lu\n", surplus_huge_pages);
2606 HSTATE_ATTR_RO(surplus_hugepages);
2608 static struct attribute *hstate_attrs[] = {
2609 &nr_hugepages_attr.attr,
2610 &nr_overcommit_hugepages_attr.attr,
2611 &free_hugepages_attr.attr,
2612 &resv_hugepages_attr.attr,
2613 &surplus_hugepages_attr.attr,
2615 &nr_hugepages_mempolicy_attr.attr,
2620 static struct attribute_group hstate_attr_group = {
2621 .attrs = hstate_attrs,
2624 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
2625 struct kobject **hstate_kobjs,
2626 struct attribute_group *hstate_attr_group)
2629 int hi = hstate_index(h);
2631 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
2632 if (!hstate_kobjs[hi])
2635 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
2637 kobject_put(hstate_kobjs[hi]);
2642 static void __init hugetlb_sysfs_init(void)
2647 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
2648 if (!hugepages_kobj)
2651 for_each_hstate(h) {
2652 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
2653 hstate_kobjs, &hstate_attr_group);
2655 pr_err("Hugetlb: Unable to add hstate %s", h->name);
2662 * node_hstate/s - associate per node hstate attributes, via their kobjects,
2663 * with node devices in node_devices[] using a parallel array. The array
2664 * index of a node device or _hstate == node id.
2665 * This is here to avoid any static dependency of the node device driver, in
2666 * the base kernel, on the hugetlb module.
2668 struct node_hstate {
2669 struct kobject *hugepages_kobj;
2670 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2672 static struct node_hstate node_hstates[MAX_NUMNODES];
2675 * A subset of global hstate attributes for node devices
2677 static struct attribute *per_node_hstate_attrs[] = {
2678 &nr_hugepages_attr.attr,
2679 &free_hugepages_attr.attr,
2680 &surplus_hugepages_attr.attr,
2684 static struct attribute_group per_node_hstate_attr_group = {
2685 .attrs = per_node_hstate_attrs,
2689 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
2690 * Returns node id via non-NULL nidp.
2692 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2696 for (nid = 0; nid < nr_node_ids; nid++) {
2697 struct node_hstate *nhs = &node_hstates[nid];
2699 for (i = 0; i < HUGE_MAX_HSTATE; i++)
2700 if (nhs->hstate_kobjs[i] == kobj) {
2712 * Unregister hstate attributes from a single node device.
2713 * No-op if no hstate attributes attached.
2715 static void hugetlb_unregister_node(struct node *node)
2718 struct node_hstate *nhs = &node_hstates[node->dev.id];
2720 if (!nhs->hugepages_kobj)
2721 return; /* no hstate attributes */
2723 for_each_hstate(h) {
2724 int idx = hstate_index(h);
2725 if (nhs->hstate_kobjs[idx]) {
2726 kobject_put(nhs->hstate_kobjs[idx]);
2727 nhs->hstate_kobjs[idx] = NULL;
2731 kobject_put(nhs->hugepages_kobj);
2732 nhs->hugepages_kobj = NULL;
2737 * Register hstate attributes for a single node device.
2738 * No-op if attributes already registered.
2740 static void hugetlb_register_node(struct node *node)
2743 struct node_hstate *nhs = &node_hstates[node->dev.id];
2746 if (nhs->hugepages_kobj)
2747 return; /* already allocated */
2749 nhs->hugepages_kobj = kobject_create_and_add("hugepages",
2751 if (!nhs->hugepages_kobj)
2754 for_each_hstate(h) {
2755 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2757 &per_node_hstate_attr_group);
2759 pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2760 h->name, node->dev.id);
2761 hugetlb_unregister_node(node);
2768 * hugetlb init time: register hstate attributes for all registered node
2769 * devices of nodes that have memory. All on-line nodes should have
2770 * registered their associated device by this time.
2772 static void __init hugetlb_register_all_nodes(void)
2776 for_each_node_state(nid, N_MEMORY) {
2777 struct node *node = node_devices[nid];
2778 if (node->dev.id == nid)
2779 hugetlb_register_node(node);
2783 * Let the node device driver know we're here so it can
2784 * [un]register hstate attributes on node hotplug.
2786 register_hugetlbfs_with_node(hugetlb_register_node,
2787 hugetlb_unregister_node);
2789 #else /* !CONFIG_NUMA */
2791 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2799 static void hugetlb_register_all_nodes(void) { }
2803 static int __init hugetlb_init(void)
2807 if (!hugepages_supported())
2810 if (!size_to_hstate(default_hstate_size)) {
2811 default_hstate_size = HPAGE_SIZE;
2812 if (!size_to_hstate(default_hstate_size))
2813 hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
2815 default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
2816 if (default_hstate_max_huge_pages) {
2817 if (!default_hstate.max_huge_pages)
2818 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
2821 hugetlb_init_hstates();
2822 gather_bootmem_prealloc();
2825 hugetlb_sysfs_init();
2826 hugetlb_register_all_nodes();
2827 hugetlb_cgroup_file_init();
2830 num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2832 num_fault_mutexes = 1;
2834 hugetlb_fault_mutex_table =
2835 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2836 BUG_ON(!hugetlb_fault_mutex_table);
2838 for (i = 0; i < num_fault_mutexes; i++)
2839 mutex_init(&hugetlb_fault_mutex_table[i]);
2842 subsys_initcall(hugetlb_init);
2844 /* Should be called on processing a hugepagesz=... option */
2845 void __init hugetlb_bad_size(void)
2847 parsed_valid_hugepagesz = false;
2850 void __init hugetlb_add_hstate(unsigned int order)
2855 if (size_to_hstate(PAGE_SIZE << order)) {
2856 pr_warn("hugepagesz= specified twice, ignoring\n");
2859 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2861 h = &hstates[hugetlb_max_hstate++];
2863 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2864 h->nr_huge_pages = 0;
2865 h->free_huge_pages = 0;
2866 for (i = 0; i < MAX_NUMNODES; ++i)
2867 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2868 INIT_LIST_HEAD(&h->hugepage_activelist);
2869 h->next_nid_to_alloc = first_memory_node;
2870 h->next_nid_to_free = first_memory_node;
2871 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2872 huge_page_size(h)/1024);
2877 static int __init hugetlb_nrpages_setup(char *s)
2880 static unsigned long *last_mhp;
2882 if (!parsed_valid_hugepagesz) {
2883 pr_warn("hugepages = %s preceded by "
2884 "an unsupported hugepagesz, ignoring\n", s);
2885 parsed_valid_hugepagesz = true;
2889 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2890 * so this hugepages= parameter goes to the "default hstate".
2892 else if (!hugetlb_max_hstate)
2893 mhp = &default_hstate_max_huge_pages;
2895 mhp = &parsed_hstate->max_huge_pages;
2897 if (mhp == last_mhp) {
2898 pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n");
2902 if (sscanf(s, "%lu", mhp) <= 0)
2906 * Global state is always initialized later in hugetlb_init.
2907 * But we need to allocate >= MAX_ORDER hstates here early to still
2908 * use the bootmem allocator.
2910 if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2911 hugetlb_hstate_alloc_pages(parsed_hstate);
2917 __setup("hugepages=", hugetlb_nrpages_setup);
2919 static int __init hugetlb_default_setup(char *s)
2921 default_hstate_size = memparse(s, &s);
2924 __setup("default_hugepagesz=", hugetlb_default_setup);
2926 static unsigned int cpuset_mems_nr(unsigned int *array)
2929 unsigned int nr = 0;
2931 for_each_node_mask(node, cpuset_current_mems_allowed)
2937 #ifdef CONFIG_SYSCTL
2938 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2939 struct ctl_table *table, int write,
2940 void __user *buffer, size_t *length, loff_t *ppos)
2942 struct hstate *h = &default_hstate;
2943 unsigned long tmp = h->max_huge_pages;
2946 if (!hugepages_supported())
2950 table->maxlen = sizeof(unsigned long);
2951 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2956 ret = __nr_hugepages_store_common(obey_mempolicy, h,
2957 NUMA_NO_NODE, tmp, *length);
2962 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2963 void __user *buffer, size_t *length, loff_t *ppos)
2966 return hugetlb_sysctl_handler_common(false, table, write,
2967 buffer, length, ppos);
2971 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2972 void __user *buffer, size_t *length, loff_t *ppos)
2974 return hugetlb_sysctl_handler_common(true, table, write,
2975 buffer, length, ppos);
2977 #endif /* CONFIG_NUMA */
2979 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2980 void __user *buffer,
2981 size_t *length, loff_t *ppos)
2983 struct hstate *h = &default_hstate;
2987 if (!hugepages_supported())
2990 tmp = h->nr_overcommit_huge_pages;
2992 if (write && hstate_is_gigantic(h))
2996 table->maxlen = sizeof(unsigned long);
2997 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
3002 spin_lock(&hugetlb_lock);
3003 h->nr_overcommit_huge_pages = tmp;
3004 spin_unlock(&hugetlb_lock);
3010 #endif /* CONFIG_SYSCTL */
3012 void hugetlb_report_meminfo(struct seq_file *m)
3014 struct hstate *h = &default_hstate;
3015 if (!hugepages_supported())
3018 "HugePages_Total: %5lu\n"
3019 "HugePages_Free: %5lu\n"
3020 "HugePages_Rsvd: %5lu\n"
3021 "HugePages_Surp: %5lu\n"
3022 "Hugepagesize: %8lu kB\n",
3026 h->surplus_huge_pages,
3027 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
3030 int hugetlb_report_node_meminfo(int nid, char *buf)
3032 struct hstate *h = &default_hstate;
3033 if (!hugepages_supported())
3036 "Node %d HugePages_Total: %5u\n"
3037 "Node %d HugePages_Free: %5u\n"
3038 "Node %d HugePages_Surp: %5u\n",
3039 nid, h->nr_huge_pages_node[nid],
3040 nid, h->free_huge_pages_node[nid],
3041 nid, h->surplus_huge_pages_node[nid]);
3044 void hugetlb_show_meminfo(void)
3049 if (!hugepages_supported())
3052 for_each_node_state(nid, N_MEMORY)
3054 pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
3056 h->nr_huge_pages_node[nid],
3057 h->free_huge_pages_node[nid],
3058 h->surplus_huge_pages_node[nid],
3059 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
3062 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
3064 seq_printf(m, "HugetlbPages:\t%8lu kB\n",
3065 atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10));
3068 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
3069 unsigned long hugetlb_total_pages(void)
3072 unsigned long nr_total_pages = 0;
3075 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
3076 return nr_total_pages;
3079 static int hugetlb_acct_memory(struct hstate *h, long delta)
3083 spin_lock(&hugetlb_lock);
3085 * When cpuset is configured, it breaks the strict hugetlb page
3086 * reservation as the accounting is done on a global variable. Such
3087 * reservation is completely rubbish in the presence of cpuset because
3088 * the reservation is not checked against page availability for the
3089 * current cpuset. Application can still potentially OOM'ed by kernel
3090 * with lack of free htlb page in cpuset that the task is in.
3091 * Attempt to enforce strict accounting with cpuset is almost
3092 * impossible (or too ugly) because cpuset is too fluid that
3093 * task or memory node can be dynamically moved between cpusets.
3095 * The change of semantics for shared hugetlb mapping with cpuset is
3096 * undesirable. However, in order to preserve some of the semantics,
3097 * we fall back to check against current free page availability as
3098 * a best attempt and hopefully to minimize the impact of changing
3099 * semantics that cpuset has.
3102 if (gather_surplus_pages(h, delta) < 0)
3105 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
3106 return_unused_surplus_pages(h, delta);
3113 return_unused_surplus_pages(h, (unsigned long) -delta);
3116 spin_unlock(&hugetlb_lock);
3120 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
3122 struct resv_map *resv = vma_resv_map(vma);
3125 * This new VMA should share its siblings reservation map if present.
3126 * The VMA will only ever have a valid reservation map pointer where
3127 * it is being copied for another still existing VMA. As that VMA
3128 * has a reference to the reservation map it cannot disappear until
3129 * after this open call completes. It is therefore safe to take a
3130 * new reference here without additional locking.
3132 if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3133 kref_get(&resv->refs);
3136 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
3138 struct hstate *h = hstate_vma(vma);
3139 struct resv_map *resv = vma_resv_map(vma);
3140 struct hugepage_subpool *spool = subpool_vma(vma);
3141 unsigned long reserve, start, end;
3144 if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3147 start = vma_hugecache_offset(h, vma, vma->vm_start);
3148 end = vma_hugecache_offset(h, vma, vma->vm_end);
3150 reserve = (end - start) - region_count(resv, start, end);
3152 kref_put(&resv->refs, resv_map_release);
3156 * Decrement reserve counts. The global reserve count may be
3157 * adjusted if the subpool has a minimum size.
3159 gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
3160 hugetlb_acct_memory(h, -gbl_reserve);
3165 * We cannot handle pagefaults against hugetlb pages at all. They cause
3166 * handle_mm_fault() to try to instantiate regular-sized pages in the
3167 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
3170 static int hugetlb_vm_op_fault(struct vm_fault *vmf)
3176 const struct vm_operations_struct hugetlb_vm_ops = {
3177 .fault = hugetlb_vm_op_fault,
3178 .open = hugetlb_vm_op_open,
3179 .close = hugetlb_vm_op_close,
3182 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
3188 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
3189 vma->vm_page_prot)));
3191 entry = huge_pte_wrprotect(mk_huge_pte(page,
3192 vma->vm_page_prot));
3194 entry = pte_mkyoung(entry);
3195 entry = pte_mkhuge(entry);
3196 entry = arch_make_huge_pte(entry, vma, page, writable);
3201 static void set_huge_ptep_writable(struct vm_area_struct *vma,
3202 unsigned long address, pte_t *ptep)
3206 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
3207 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
3208 update_mmu_cache(vma, address, ptep);
3211 bool is_hugetlb_entry_migration(pte_t pte)
3215 if (huge_pte_none(pte) || pte_present(pte))
3217 swp = pte_to_swp_entry(pte);
3218 if (non_swap_entry(swp) && is_migration_entry(swp))
3224 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
3228 if (huge_pte_none(pte) || pte_present(pte))
3230 swp = pte_to_swp_entry(pte);
3231 if (non_swap_entry(swp) && is_hwpoison_entry(swp))
3237 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
3238 struct vm_area_struct *vma)
3240 pte_t *src_pte, *dst_pte, entry;
3241 struct page *ptepage;
3244 struct hstate *h = hstate_vma(vma);
3245 unsigned long sz = huge_page_size(h);
3246 unsigned long mmun_start; /* For mmu_notifiers */
3247 unsigned long mmun_end; /* For mmu_notifiers */
3250 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
3252 mmun_start = vma->vm_start;
3253 mmun_end = vma->vm_end;
3255 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
3257 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
3258 spinlock_t *src_ptl, *dst_ptl;
3259 src_pte = huge_pte_offset(src, addr, sz);
3262 dst_pte = huge_pte_alloc(dst, addr, sz);
3268 /* If the pagetables are shared don't copy or take references */
3269 if (dst_pte == src_pte)
3272 dst_ptl = huge_pte_lock(h, dst, dst_pte);
3273 src_ptl = huge_pte_lockptr(h, src, src_pte);
3274 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
3275 entry = huge_ptep_get(src_pte);
3276 if (huge_pte_none(entry)) { /* skip none entry */
3278 } else if (unlikely(is_hugetlb_entry_migration(entry) ||
3279 is_hugetlb_entry_hwpoisoned(entry))) {
3280 swp_entry_t swp_entry = pte_to_swp_entry(entry);
3282 if (is_write_migration_entry(swp_entry) && cow) {
3284 * COW mappings require pages in both
3285 * parent and child to be set to read.
3287 make_migration_entry_read(&swp_entry);
3288 entry = swp_entry_to_pte(swp_entry);
3289 set_huge_swap_pte_at(src, addr, src_pte,
3292 set_huge_swap_pte_at(dst, addr, dst_pte, entry, sz);
3295 huge_ptep_set_wrprotect(src, addr, src_pte);
3296 mmu_notifier_invalidate_range(src, mmun_start,
3299 entry = huge_ptep_get(src_pte);
3300 ptepage = pte_page(entry);
3302 page_dup_rmap(ptepage, true);
3303 set_huge_pte_at(dst, addr, dst_pte, entry);
3304 hugetlb_count_add(pages_per_huge_page(h), dst);
3306 spin_unlock(src_ptl);
3307 spin_unlock(dst_ptl);
3311 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
3316 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
3317 unsigned long start, unsigned long end,
3318 struct page *ref_page)
3320 struct mm_struct *mm = vma->vm_mm;
3321 unsigned long address;
3326 struct hstate *h = hstate_vma(vma);
3327 unsigned long sz = huge_page_size(h);
3328 const unsigned long mmun_start = start; /* For mmu_notifiers */
3329 const unsigned long mmun_end = end; /* For mmu_notifiers */
3331 WARN_ON(!is_vm_hugetlb_page(vma));
3332 BUG_ON(start & ~huge_page_mask(h));
3333 BUG_ON(end & ~huge_page_mask(h));
3336 * This is a hugetlb vma, all the pte entries should point
3339 tlb_remove_check_page_size_change(tlb, sz);
3340 tlb_start_vma(tlb, vma);
3341 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3343 for (; address < end; address += sz) {
3344 ptep = huge_pte_offset(mm, address, sz);
3348 ptl = huge_pte_lock(h, mm, ptep);
3349 if (huge_pmd_unshare(mm, &address, ptep)) {
3354 pte = huge_ptep_get(ptep);
3355 if (huge_pte_none(pte)) {
3361 * Migrating hugepage or HWPoisoned hugepage is already
3362 * unmapped and its refcount is dropped, so just clear pte here.
3364 if (unlikely(!pte_present(pte))) {
3365 huge_pte_clear(mm, address, ptep, sz);
3370 page = pte_page(pte);
3372 * If a reference page is supplied, it is because a specific
3373 * page is being unmapped, not a range. Ensure the page we
3374 * are about to unmap is the actual page of interest.
3377 if (page != ref_page) {
3382 * Mark the VMA as having unmapped its page so that
3383 * future faults in this VMA will fail rather than
3384 * looking like data was lost
3386 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
3389 pte = huge_ptep_get_and_clear(mm, address, ptep);
3390 tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
3391 if (huge_pte_dirty(pte))
3392 set_page_dirty(page);
3394 hugetlb_count_sub(pages_per_huge_page(h), mm);
3395 page_remove_rmap(page, true);
3398 tlb_remove_page_size(tlb, page, huge_page_size(h));
3400 * Bail out after unmapping reference page if supplied
3405 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3406 tlb_end_vma(tlb, vma);
3409 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
3410 struct vm_area_struct *vma, unsigned long start,
3411 unsigned long end, struct page *ref_page)
3413 __unmap_hugepage_range(tlb, vma, start, end, ref_page);
3416 * Clear this flag so that x86's huge_pmd_share page_table_shareable
3417 * test will fail on a vma being torn down, and not grab a page table
3418 * on its way out. We're lucky that the flag has such an appropriate
3419 * name, and can in fact be safely cleared here. We could clear it
3420 * before the __unmap_hugepage_range above, but all that's necessary
3421 * is to clear it before releasing the i_mmap_rwsem. This works
3422 * because in the context this is called, the VMA is about to be
3423 * destroyed and the i_mmap_rwsem is held.
3425 vma->vm_flags &= ~VM_MAYSHARE;
3428 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
3429 unsigned long end, struct page *ref_page)
3431 struct mm_struct *mm;
3432 struct mmu_gather tlb;
3436 tlb_gather_mmu(&tlb, mm, start, end);
3437 __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
3438 tlb_finish_mmu(&tlb, start, end);
3442 * This is called when the original mapper is failing to COW a MAP_PRIVATE
3443 * mappping it owns the reserve page for. The intention is to unmap the page
3444 * from other VMAs and let the children be SIGKILLed if they are faulting the
3447 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
3448 struct page *page, unsigned long address)
3450 struct hstate *h = hstate_vma(vma);
3451 struct vm_area_struct *iter_vma;
3452 struct address_space *mapping;
3456 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
3457 * from page cache lookup which is in HPAGE_SIZE units.
3459 address = address & huge_page_mask(h);
3460 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
3462 mapping = vma->vm_file->f_mapping;
3465 * Take the mapping lock for the duration of the table walk. As
3466 * this mapping should be shared between all the VMAs,
3467 * __unmap_hugepage_range() is called as the lock is already held
3469 i_mmap_lock_write(mapping);
3470 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
3471 /* Do not unmap the current VMA */
3472 if (iter_vma == vma)
3476 * Shared VMAs have their own reserves and do not affect
3477 * MAP_PRIVATE accounting but it is possible that a shared
3478 * VMA is using the same page so check and skip such VMAs.
3480 if (iter_vma->vm_flags & VM_MAYSHARE)
3484 * Unmap the page from other VMAs without their own reserves.
3485 * They get marked to be SIGKILLed if they fault in these
3486 * areas. This is because a future no-page fault on this VMA
3487 * could insert a zeroed page instead of the data existing
3488 * from the time of fork. This would look like data corruption
3490 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
3491 unmap_hugepage_range(iter_vma, address,
3492 address + huge_page_size(h), page);
3494 i_mmap_unlock_write(mapping);
3498 * Hugetlb_cow() should be called with page lock of the original hugepage held.
3499 * Called with hugetlb_instantiation_mutex held and pte_page locked so we
3500 * cannot race with other handlers or page migration.
3501 * Keep the pte_same checks anyway to make transition from the mutex easier.
3503 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
3504 unsigned long address, pte_t *ptep,
3505 struct page *pagecache_page, spinlock_t *ptl)
3508 struct hstate *h = hstate_vma(vma);
3509 struct page *old_page, *new_page;
3510 int ret = 0, outside_reserve = 0;
3511 unsigned long mmun_start; /* For mmu_notifiers */
3512 unsigned long mmun_end; /* For mmu_notifiers */
3514 pte = huge_ptep_get(ptep);
3515 old_page = pte_page(pte);
3518 /* If no-one else is actually using this page, avoid the copy
3519 * and just make the page writable */
3520 if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
3521 page_move_anon_rmap(old_page, vma);
3522 set_huge_ptep_writable(vma, address, ptep);
3527 * If the process that created a MAP_PRIVATE mapping is about to
3528 * perform a COW due to a shared page count, attempt to satisfy
3529 * the allocation without using the existing reserves. The pagecache
3530 * page is used to determine if the reserve at this address was
3531 * consumed or not. If reserves were used, a partial faulted mapping
3532 * at the time of fork() could consume its reserves on COW instead
3533 * of the full address range.
3535 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
3536 old_page != pagecache_page)
3537 outside_reserve = 1;
3542 * Drop page table lock as buddy allocator may be called. It will
3543 * be acquired again before returning to the caller, as expected.
3546 new_page = alloc_huge_page(vma, address, outside_reserve);
3548 if (IS_ERR(new_page)) {
3550 * If a process owning a MAP_PRIVATE mapping fails to COW,
3551 * it is due to references held by a child and an insufficient
3552 * huge page pool. To guarantee the original mappers
3553 * reliability, unmap the page from child processes. The child
3554 * may get SIGKILLed if it later faults.
3556 if (outside_reserve) {
3558 BUG_ON(huge_pte_none(pte));
3559 unmap_ref_private(mm, vma, old_page, address);
3560 BUG_ON(huge_pte_none(pte));
3562 ptep = huge_pte_offset(mm, address & huge_page_mask(h),
3565 pte_same(huge_ptep_get(ptep), pte)))
3566 goto retry_avoidcopy;
3568 * race occurs while re-acquiring page table
3569 * lock, and our job is done.
3574 ret = (PTR_ERR(new_page) == -ENOMEM) ?
3575 VM_FAULT_OOM : VM_FAULT_SIGBUS;
3576 goto out_release_old;
3580 * When the original hugepage is shared one, it does not have
3581 * anon_vma prepared.
3583 if (unlikely(anon_vma_prepare(vma))) {
3585 goto out_release_all;
3588 copy_user_huge_page(new_page, old_page, address, vma,
3589 pages_per_huge_page(h));
3590 __SetPageUptodate(new_page);
3591 set_page_huge_active(new_page);
3593 mmun_start = address & huge_page_mask(h);
3594 mmun_end = mmun_start + huge_page_size(h);
3595 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3598 * Retake the page table lock to check for racing updates
3599 * before the page tables are altered
3602 ptep = huge_pte_offset(mm, address & huge_page_mask(h),
3604 if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
3605 ClearPagePrivate(new_page);
3608 huge_ptep_clear_flush(vma, address, ptep);
3609 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
3610 set_huge_pte_at(mm, address, ptep,
3611 make_huge_pte(vma, new_page, 1));
3612 page_remove_rmap(old_page, true);
3613 hugepage_add_new_anon_rmap(new_page, vma, address);
3614 /* Make the old page be freed below */
3615 new_page = old_page;
3618 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3620 restore_reserve_on_error(h, vma, address, new_page);
3625 spin_lock(ptl); /* Caller expects lock to be held */
3629 /* Return the pagecache page at a given address within a VMA */
3630 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
3631 struct vm_area_struct *vma, unsigned long address)
3633 struct address_space *mapping;
3636 mapping = vma->vm_file->f_mapping;
3637 idx = vma_hugecache_offset(h, vma, address);
3639 return find_lock_page(mapping, idx);
3643 * Return whether there is a pagecache page to back given address within VMA.
3644 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
3646 static bool hugetlbfs_pagecache_present(struct hstate *h,
3647 struct vm_area_struct *vma, unsigned long address)
3649 struct address_space *mapping;
3653 mapping = vma->vm_file->f_mapping;
3654 idx = vma_hugecache_offset(h, vma, address);
3656 page = find_get_page(mapping, idx);
3659 return page != NULL;
3662 int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
3665 struct inode *inode = mapping->host;
3666 struct hstate *h = hstate_inode(inode);
3667 int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3671 ClearPagePrivate(page);
3673 spin_lock(&inode->i_lock);
3674 inode->i_blocks += blocks_per_huge_page(h);
3675 spin_unlock(&inode->i_lock);
3679 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
3680 struct address_space *mapping, pgoff_t idx,
3681 unsigned long address, pte_t *ptep, unsigned int flags)
3683 struct hstate *h = hstate_vma(vma);
3684 int ret = VM_FAULT_SIGBUS;
3692 * Currently, we are forced to kill the process in the event the
3693 * original mapper has unmapped pages from the child due to a failed
3694 * COW. Warn that such a situation has occurred as it may not be obvious
3696 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
3697 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
3703 * Use page lock to guard against racing truncation
3704 * before we get page_table_lock.
3707 page = find_lock_page(mapping, idx);
3709 size = i_size_read(mapping->host) >> huge_page_shift(h);
3714 * Check for page in userfault range
3716 if (userfaultfd_missing(vma)) {
3718 struct vm_fault vmf = {
3723 * Hard to debug if it ends up being
3724 * used by a callee that assumes
3725 * something about the other
3726 * uninitialized fields... same as in
3732 * hugetlb_fault_mutex must be dropped before
3733 * handling userfault. Reacquire after handling
3734 * fault to make calling code simpler.
3736 hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping,
3738 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
3739 ret = handle_userfault(&vmf, VM_UFFD_MISSING);
3740 mutex_lock(&hugetlb_fault_mutex_table[hash]);
3744 page = alloc_huge_page(vma, address, 0);
3746 ret = PTR_ERR(page);
3750 ret = VM_FAULT_SIGBUS;
3753 clear_huge_page(page, address, pages_per_huge_page(h));
3754 __SetPageUptodate(page);
3755 set_page_huge_active(page);
3757 if (vma->vm_flags & VM_MAYSHARE) {
3758 int err = huge_add_to_page_cache(page, mapping, idx);
3767 if (unlikely(anon_vma_prepare(vma))) {
3769 goto backout_unlocked;
3775 * If memory error occurs between mmap() and fault, some process
3776 * don't have hwpoisoned swap entry for errored virtual address.
3777 * So we need to block hugepage fault by PG_hwpoison bit check.
3779 if (unlikely(PageHWPoison(page))) {
3780 ret = VM_FAULT_HWPOISON |
3781 VM_FAULT_SET_HINDEX(hstate_index(h));
3782 goto backout_unlocked;
3787 * If we are going to COW a private mapping later, we examine the
3788 * pending reservations for this page now. This will ensure that
3789 * any allocations necessary to record that reservation occur outside
3792 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3793 if (vma_needs_reservation(h, vma, address) < 0) {
3795 goto backout_unlocked;
3797 /* Just decrements count, does not deallocate */
3798 vma_end_reservation(h, vma, address);
3801 ptl = huge_pte_lock(h, mm, ptep);
3802 size = i_size_read(mapping->host) >> huge_page_shift(h);
3807 if (!huge_pte_none(huge_ptep_get(ptep)))
3811 ClearPagePrivate(page);
3812 hugepage_add_new_anon_rmap(page, vma, address);
3814 page_dup_rmap(page, true);
3815 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
3816 && (vma->vm_flags & VM_SHARED)));
3817 set_huge_pte_at(mm, address, ptep, new_pte);
3819 hugetlb_count_add(pages_per_huge_page(h), mm);
3820 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3821 /* Optimization, do the COW without a second fault */
3822 ret = hugetlb_cow(mm, vma, address, ptep, page, ptl);
3834 restore_reserve_on_error(h, vma, address, page);
3840 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3841 struct vm_area_struct *vma,
3842 struct address_space *mapping,
3843 pgoff_t idx, unsigned long address)
3845 unsigned long key[2];
3848 if (vma->vm_flags & VM_SHARED) {
3849 key[0] = (unsigned long) mapping;
3852 key[0] = (unsigned long) mm;
3853 key[1] = address >> huge_page_shift(h);
3856 hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
3858 return hash & (num_fault_mutexes - 1);
3862 * For uniprocesor systems we always use a single mutex, so just
3863 * return 0 and avoid the hashing overhead.
3865 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3866 struct vm_area_struct *vma,
3867 struct address_space *mapping,
3868 pgoff_t idx, unsigned long address)
3874 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
3875 unsigned long address, unsigned int flags)
3882 struct page *page = NULL;
3883 struct page *pagecache_page = NULL;
3884 struct hstate *h = hstate_vma(vma);
3885 struct address_space *mapping;
3886 int need_wait_lock = 0;
3888 address &= huge_page_mask(h);
3890 ptep = huge_pte_offset(mm, address, huge_page_size(h));
3892 entry = huge_ptep_get(ptep);
3893 if (unlikely(is_hugetlb_entry_migration(entry))) {
3894 migration_entry_wait_huge(vma, mm, ptep);
3896 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
3897 return VM_FAULT_HWPOISON_LARGE |
3898 VM_FAULT_SET_HINDEX(hstate_index(h));
3900 ptep = huge_pte_alloc(mm, address, huge_page_size(h));
3902 return VM_FAULT_OOM;
3905 mapping = vma->vm_file->f_mapping;
3906 idx = vma_hugecache_offset(h, vma, address);
3909 * Serialize hugepage allocation and instantiation, so that we don't
3910 * get spurious allocation failures if two CPUs race to instantiate
3911 * the same page in the page cache.
3913 hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, address);
3914 mutex_lock(&hugetlb_fault_mutex_table[hash]);
3916 entry = huge_ptep_get(ptep);
3917 if (huge_pte_none(entry)) {
3918 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
3925 * entry could be a migration/hwpoison entry at this point, so this
3926 * check prevents the kernel from going below assuming that we have
3927 * a active hugepage in pagecache. This goto expects the 2nd page fault,
3928 * and is_hugetlb_entry_(migration|hwpoisoned) check will properly
3931 if (!pte_present(entry))
3935 * If we are going to COW the mapping later, we examine the pending
3936 * reservations for this page now. This will ensure that any
3937 * allocations necessary to record that reservation occur outside the
3938 * spinlock. For private mappings, we also lookup the pagecache
3939 * page now as it is used to determine if a reservation has been
3942 if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
3943 if (vma_needs_reservation(h, vma, address) < 0) {
3947 /* Just decrements count, does not deallocate */
3948 vma_end_reservation(h, vma, address);
3950 if (!(vma->vm_flags & VM_MAYSHARE))
3951 pagecache_page = hugetlbfs_pagecache_page(h,
3955 ptl = huge_pte_lock(h, mm, ptep);
3957 /* Check for a racing update before calling hugetlb_cow */
3958 if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3962 * hugetlb_cow() requires page locks of pte_page(entry) and
3963 * pagecache_page, so here we need take the former one
3964 * when page != pagecache_page or !pagecache_page.
3966 page = pte_page(entry);
3967 if (page != pagecache_page)
3968 if (!trylock_page(page)) {
3975 if (flags & FAULT_FLAG_WRITE) {
3976 if (!huge_pte_write(entry)) {
3977 ret = hugetlb_cow(mm, vma, address, ptep,
3978 pagecache_page, ptl);
3981 entry = huge_pte_mkdirty(entry);
3983 entry = pte_mkyoung(entry);
3984 if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3985 flags & FAULT_FLAG_WRITE))
3986 update_mmu_cache(vma, address, ptep);
3988 if (page != pagecache_page)
3994 if (pagecache_page) {
3995 unlock_page(pagecache_page);
3996 put_page(pagecache_page);
3999 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
4001 * Generally it's safe to hold refcount during waiting page lock. But
4002 * here we just wait to defer the next page fault to avoid busy loop and
4003 * the page is not used after unlocked before returning from the current
4004 * page fault. So we are safe from accessing freed page, even if we wait
4005 * here without taking refcount.
4008 wait_on_page_locked(page);
4013 * Used by userfaultfd UFFDIO_COPY. Based on mcopy_atomic_pte with
4014 * modifications for huge pages.
4016 int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
4018 struct vm_area_struct *dst_vma,
4019 unsigned long dst_addr,
4020 unsigned long src_addr,
4021 struct page **pagep)
4023 int vm_shared = dst_vma->vm_flags & VM_SHARED;
4024 struct hstate *h = hstate_vma(dst_vma);
4032 page = alloc_huge_page(dst_vma, dst_addr, 0);
4036 ret = copy_huge_page_from_user(page,
4037 (const void __user *) src_addr,
4038 pages_per_huge_page(h), false);
4040 /* fallback to copy_from_user outside mmap_sem */
4041 if (unlikely(ret)) {
4044 /* don't free the page */
4053 * The memory barrier inside __SetPageUptodate makes sure that
4054 * preceding stores to the page contents become visible before
4055 * the set_pte_at() write.
4057 __SetPageUptodate(page);
4058 set_page_huge_active(page);
4061 * If shared, add to page cache
4064 struct address_space *mapping = dst_vma->vm_file->f_mapping;
4065 pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr);
4067 ret = huge_add_to_page_cache(page, mapping, idx);
4069 goto out_release_nounlock;
4072 ptl = huge_pte_lockptr(h, dst_mm, dst_pte);
4076 if (!huge_pte_none(huge_ptep_get(dst_pte)))
4077 goto out_release_unlock;
4080 page_dup_rmap(page, true);
4082 ClearPagePrivate(page);
4083 hugepage_add_new_anon_rmap(page, dst_vma, dst_addr);
4086 _dst_pte = make_huge_pte(dst_vma, page, dst_vma->vm_flags & VM_WRITE);
4087 if (dst_vma->vm_flags & VM_WRITE)
4088 _dst_pte = huge_pte_mkdirty(_dst_pte);
4089 _dst_pte = pte_mkyoung(_dst_pte);
4091 set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
4093 (void)huge_ptep_set_access_flags(dst_vma, dst_addr, dst_pte, _dst_pte,
4094 dst_vma->vm_flags & VM_WRITE);
4095 hugetlb_count_add(pages_per_huge_page(h), dst_mm);
4097 /* No need to invalidate - it was non-present before */
4098 update_mmu_cache(dst_vma, dst_addr, dst_pte);
4108 out_release_nounlock:
4115 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
4116 struct page **pages, struct vm_area_struct **vmas,
4117 unsigned long *position, unsigned long *nr_pages,
4118 long i, unsigned int flags, int *nonblocking)
4120 unsigned long pfn_offset;
4121 unsigned long vaddr = *position;
4122 unsigned long remainder = *nr_pages;
4123 struct hstate *h = hstate_vma(vma);
4125 while (vaddr < vma->vm_end && remainder) {
4127 spinlock_t *ptl = NULL;
4132 * If we have a pending SIGKILL, don't keep faulting pages and
4133 * potentially allocating memory.
4135 if (unlikely(fatal_signal_pending(current))) {
4141 * Some archs (sparc64, sh*) have multiple pte_ts to
4142 * each hugepage. We have to make sure we get the
4143 * first, for the page indexing below to work.
4145 * Note that page table lock is not held when pte is null.
4147 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h),
4150 ptl = huge_pte_lock(h, mm, pte);
4151 absent = !pte || huge_pte_none(huge_ptep_get(pte));
4154 * When coredumping, it suits get_dump_page if we just return
4155 * an error where there's an empty slot with no huge pagecache
4156 * to back it. This way, we avoid allocating a hugepage, and
4157 * the sparse dumpfile avoids allocating disk blocks, but its
4158 * huge holes still show up with zeroes where they need to be.
4160 if (absent && (flags & FOLL_DUMP) &&
4161 !hugetlbfs_pagecache_present(h, vma, vaddr)) {
4169 * We need call hugetlb_fault for both hugepages under migration
4170 * (in which case hugetlb_fault waits for the migration,) and
4171 * hwpoisoned hugepages (in which case we need to prevent the
4172 * caller from accessing to them.) In order to do this, we use
4173 * here is_swap_pte instead of is_hugetlb_entry_migration and
4174 * is_hugetlb_entry_hwpoisoned. This is because it simply covers
4175 * both cases, and because we can't follow correct pages
4176 * directly from any kind of swap entries.
4178 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
4179 ((flags & FOLL_WRITE) &&
4180 !huge_pte_write(huge_ptep_get(pte)))) {
4182 unsigned int fault_flags = 0;
4186 if (flags & FOLL_WRITE)
4187 fault_flags |= FAULT_FLAG_WRITE;
4189 fault_flags |= FAULT_FLAG_ALLOW_RETRY;
4190 if (flags & FOLL_NOWAIT)
4191 fault_flags |= FAULT_FLAG_ALLOW_RETRY |
4192 FAULT_FLAG_RETRY_NOWAIT;
4193 if (flags & FOLL_TRIED) {
4194 VM_WARN_ON_ONCE(fault_flags &
4195 FAULT_FLAG_ALLOW_RETRY);
4196 fault_flags |= FAULT_FLAG_TRIED;
4198 ret = hugetlb_fault(mm, vma, vaddr, fault_flags);
4199 if (ret & VM_FAULT_ERROR) {
4200 int err = vm_fault_to_errno(ret, flags);
4208 if (ret & VM_FAULT_RETRY) {
4213 * VM_FAULT_RETRY must not return an
4214 * error, it will return zero
4217 * No need to update "position" as the
4218 * caller will not check it after
4219 * *nr_pages is set to 0.
4226 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
4227 page = pte_page(huge_ptep_get(pte));
4230 pages[i] = mem_map_offset(page, pfn_offset);
4241 if (vaddr < vma->vm_end && remainder &&
4242 pfn_offset < pages_per_huge_page(h)) {
4244 * We use pfn_offset to avoid touching the pageframes
4245 * of this compound page.
4251 *nr_pages = remainder;
4253 * setting position is actually required only if remainder is
4254 * not zero but it's faster not to add a "if (remainder)"
4259 return i ? i : -EFAULT;
4262 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
4264 * ARCHes with special requirements for evicting HUGETLB backing TLB entries can
4267 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
4270 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
4271 unsigned long address, unsigned long end, pgprot_t newprot)
4273 struct mm_struct *mm = vma->vm_mm;
4274 unsigned long start = address;
4277 struct hstate *h = hstate_vma(vma);
4278 unsigned long pages = 0;
4280 BUG_ON(address >= end);
4281 flush_cache_range(vma, address, end);
4283 mmu_notifier_invalidate_range_start(mm, start, end);
4284 i_mmap_lock_write(vma->vm_file->f_mapping);
4285 for (; address < end; address += huge_page_size(h)) {
4287 ptep = huge_pte_offset(mm, address, huge_page_size(h));
4290 ptl = huge_pte_lock(h, mm, ptep);
4291 if (huge_pmd_unshare(mm, &address, ptep)) {
4296 pte = huge_ptep_get(ptep);
4297 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
4301 if (unlikely(is_hugetlb_entry_migration(pte))) {
4302 swp_entry_t entry = pte_to_swp_entry(pte);
4304 if (is_write_migration_entry(entry)) {
4307 make_migration_entry_read(&entry);
4308 newpte = swp_entry_to_pte(entry);
4309 set_huge_swap_pte_at(mm, address, ptep,
4310 newpte, huge_page_size(h));
4316 if (!huge_pte_none(pte)) {
4317 pte = huge_ptep_get_and_clear(mm, address, ptep);
4318 pte = pte_mkhuge(huge_pte_modify(pte, newprot));
4319 pte = arch_make_huge_pte(pte, vma, NULL, 0);
4320 set_huge_pte_at(mm, address, ptep, pte);
4326 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
4327 * may have cleared our pud entry and done put_page on the page table:
4328 * once we release i_mmap_rwsem, another task can do the final put_page
4329 * and that page table be reused and filled with junk.
4331 flush_hugetlb_tlb_range(vma, start, end);
4332 mmu_notifier_invalidate_range(mm, start, end);
4333 i_mmap_unlock_write(vma->vm_file->f_mapping);
4334 mmu_notifier_invalidate_range_end(mm, start, end);
4336 return pages << h->order;
4339 int hugetlb_reserve_pages(struct inode *inode,
4341 struct vm_area_struct *vma,
4342 vm_flags_t vm_flags)
4345 struct hstate *h = hstate_inode(inode);
4346 struct hugepage_subpool *spool = subpool_inode(inode);
4347 struct resv_map *resv_map;
4351 * Only apply hugepage reservation if asked. At fault time, an
4352 * attempt will be made for VM_NORESERVE to allocate a page
4353 * without using reserves
4355 if (vm_flags & VM_NORESERVE)
4359 * Shared mappings base their reservation on the number of pages that
4360 * are already allocated on behalf of the file. Private mappings need
4361 * to reserve the full area even if read-only as mprotect() may be
4362 * called to make the mapping read-write. Assume !vma is a shm mapping
4364 if (!vma || vma->vm_flags & VM_MAYSHARE) {
4365 resv_map = inode_resv_map(inode);
4367 chg = region_chg(resv_map, from, to);
4370 resv_map = resv_map_alloc();
4376 set_vma_resv_map(vma, resv_map);
4377 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
4386 * There must be enough pages in the subpool for the mapping. If
4387 * the subpool has a minimum size, there may be some global
4388 * reservations already in place (gbl_reserve).
4390 gbl_reserve = hugepage_subpool_get_pages(spool, chg);
4391 if (gbl_reserve < 0) {
4397 * Check enough hugepages are available for the reservation.
4398 * Hand the pages back to the subpool if there are not
4400 ret = hugetlb_acct_memory(h, gbl_reserve);
4402 /* put back original number of pages, chg */
4403 (void)hugepage_subpool_put_pages(spool, chg);
4408 * Account for the reservations made. Shared mappings record regions
4409 * that have reservations as they are shared by multiple VMAs.
4410 * When the last VMA disappears, the region map says how much
4411 * the reservation was and the page cache tells how much of
4412 * the reservation was consumed. Private mappings are per-VMA and
4413 * only the consumed reservations are tracked. When the VMA
4414 * disappears, the original reservation is the VMA size and the
4415 * consumed reservations are stored in the map. Hence, nothing
4416 * else has to be done for private mappings here
4418 if (!vma || vma->vm_flags & VM_MAYSHARE) {
4419 long add = region_add(resv_map, from, to);
4421 if (unlikely(chg > add)) {
4423 * pages in this range were added to the reserve
4424 * map between region_chg and region_add. This
4425 * indicates a race with alloc_huge_page. Adjust
4426 * the subpool and reserve counts modified above
4427 * based on the difference.
4431 rsv_adjust = hugepage_subpool_put_pages(spool,
4433 hugetlb_acct_memory(h, -rsv_adjust);
4438 if (!vma || vma->vm_flags & VM_MAYSHARE)
4439 /* Don't call region_abort if region_chg failed */
4441 region_abort(resv_map, from, to);
4442 if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
4443 kref_put(&resv_map->refs, resv_map_release);
4447 long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
4450 struct hstate *h = hstate_inode(inode);
4451 struct resv_map *resv_map = inode_resv_map(inode);
4453 struct hugepage_subpool *spool = subpool_inode(inode);
4457 chg = region_del(resv_map, start, end);
4459 * region_del() can fail in the rare case where a region
4460 * must be split and another region descriptor can not be
4461 * allocated. If end == LONG_MAX, it will not fail.
4467 spin_lock(&inode->i_lock);
4468 inode->i_blocks -= (blocks_per_huge_page(h) * freed);
4469 spin_unlock(&inode->i_lock);
4472 * If the subpool has a minimum size, the number of global
4473 * reservations to be released may be adjusted.
4475 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
4476 hugetlb_acct_memory(h, -gbl_reserve);
4481 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
4482 static unsigned long page_table_shareable(struct vm_area_struct *svma,
4483 struct vm_area_struct *vma,
4484 unsigned long addr, pgoff_t idx)
4486 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
4488 unsigned long sbase = saddr & PUD_MASK;
4489 unsigned long s_end = sbase + PUD_SIZE;
4491 /* Allow segments to share if only one is marked locked */
4492 unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
4493 unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK;
4496 * match the virtual addresses, permission and the alignment of the
4499 if (pmd_index(addr) != pmd_index(saddr) ||
4500 vm_flags != svm_flags ||
4501 sbase < svma->vm_start || svma->vm_end < s_end)
4507 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
4509 unsigned long base = addr & PUD_MASK;
4510 unsigned long end = base + PUD_SIZE;
4513 * check on proper vm_flags and page table alignment
4515 if (vma->vm_flags & VM_MAYSHARE &&
4516 vma->vm_start <= base && end <= vma->vm_end)
4522 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
4523 * and returns the corresponding pte. While this is not necessary for the
4524 * !shared pmd case because we can allocate the pmd later as well, it makes the
4525 * code much cleaner. pmd allocation is essential for the shared case because
4526 * pud has to be populated inside the same i_mmap_rwsem section - otherwise
4527 * racing tasks could either miss the sharing (see huge_pte_offset) or select a
4528 * bad pmd for sharing.
4530 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4532 struct vm_area_struct *vma = find_vma(mm, addr);
4533 struct address_space *mapping = vma->vm_file->f_mapping;
4534 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
4536 struct vm_area_struct *svma;
4537 unsigned long saddr;
4542 if (!vma_shareable(vma, addr))
4543 return (pte_t *)pmd_alloc(mm, pud, addr);
4545 i_mmap_lock_write(mapping);
4546 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
4550 saddr = page_table_shareable(svma, vma, addr, idx);
4552 spte = huge_pte_offset(svma->vm_mm, saddr,
4553 vma_mmu_pagesize(svma));
4555 get_page(virt_to_page(spte));
4564 ptl = huge_pte_lock(hstate_vma(vma), mm, spte);
4565 if (pud_none(*pud)) {
4566 pud_populate(mm, pud,
4567 (pmd_t *)((unsigned long)spte & PAGE_MASK));
4570 put_page(virt_to_page(spte));
4574 pte = (pte_t *)pmd_alloc(mm, pud, addr);
4575 i_mmap_unlock_write(mapping);
4580 * unmap huge page backed by shared pte.
4582 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared
4583 * indicated by page_count > 1, unmap is achieved by clearing pud and
4584 * decrementing the ref count. If count == 1, the pte page is not shared.
4586 * called with page table lock held.
4588 * returns: 1 successfully unmapped a shared pte page
4589 * 0 the underlying pte page is not shared, or it is the last user
4591 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4593 pgd_t *pgd = pgd_offset(mm, *addr);
4594 p4d_t *p4d = p4d_offset(pgd, *addr);
4595 pud_t *pud = pud_offset(p4d, *addr);
4597 BUG_ON(page_count(virt_to_page(ptep)) == 0);
4598 if (page_count(virt_to_page(ptep)) == 1)
4602 put_page(virt_to_page(ptep));
4604 *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
4607 #define want_pmd_share() (1)
4608 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4609 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4614 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4618 #define want_pmd_share() (0)
4619 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4621 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
4622 pte_t *huge_pte_alloc(struct mm_struct *mm,
4623 unsigned long addr, unsigned long sz)
4630 pgd = pgd_offset(mm, addr);
4631 p4d = p4d_offset(pgd, addr);
4632 pud = pud_alloc(mm, p4d, addr);
4634 if (sz == PUD_SIZE) {
4637 BUG_ON(sz != PMD_SIZE);
4638 if (want_pmd_share() && pud_none(*pud))
4639 pte = huge_pmd_share(mm, addr, pud);
4641 pte = (pte_t *)pmd_alloc(mm, pud, addr);
4644 BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte));
4649 pte_t *huge_pte_offset(struct mm_struct *mm,
4650 unsigned long addr, unsigned long sz)
4657 pgd = pgd_offset(mm, addr);
4658 if (!pgd_present(*pgd))
4660 p4d = p4d_offset(pgd, addr);
4661 if (!p4d_present(*p4d))
4663 pud = pud_offset(p4d, addr);
4664 if (!pud_present(*pud))
4667 return (pte_t *)pud;
4668 pmd = pmd_offset(pud, addr);
4669 return (pte_t *) pmd;
4672 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
4675 * These functions are overwritable if your architecture needs its own
4678 struct page * __weak
4679 follow_huge_addr(struct mm_struct *mm, unsigned long address,
4682 return ERR_PTR(-EINVAL);
4685 struct page * __weak
4686 follow_huge_pd(struct vm_area_struct *vma,
4687 unsigned long address, hugepd_t hpd, int flags, int pdshift)
4689 WARN(1, "hugepd follow called with no support for hugepage directory format\n");
4693 struct page * __weak
4694 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
4695 pmd_t *pmd, int flags)
4697 struct page *page = NULL;
4701 ptl = pmd_lockptr(mm, pmd);
4704 * make sure that the address range covered by this pmd is not
4705 * unmapped from other threads.
4707 if (!pmd_huge(*pmd))
4709 pte = huge_ptep_get((pte_t *)pmd);
4710 if (pte_present(pte)) {
4711 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
4712 if (flags & FOLL_GET)
4715 if (is_hugetlb_entry_migration(pte)) {
4717 __migration_entry_wait(mm, (pte_t *)pmd, ptl);
4721 * hwpoisoned entry is treated as no_page_table in
4722 * follow_page_mask().
4730 struct page * __weak
4731 follow_huge_pud(struct mm_struct *mm, unsigned long address,
4732 pud_t *pud, int flags)
4734 if (flags & FOLL_GET)
4737 return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
4740 struct page * __weak
4741 follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags)
4743 if (flags & FOLL_GET)
4746 return pte_page(*(pte_t *)pgd) + ((address & ~PGDIR_MASK) >> PAGE_SHIFT);
4749 #ifdef CONFIG_MEMORY_FAILURE
4752 * This function is called from memory failure code.
4754 int dequeue_hwpoisoned_huge_page(struct page *hpage)
4756 struct hstate *h = page_hstate(hpage);
4757 int nid = page_to_nid(hpage);
4760 spin_lock(&hugetlb_lock);
4762 * Just checking !page_huge_active is not enough, because that could be
4763 * an isolated/hwpoisoned hugepage (which have >0 refcount).
4765 if (!page_huge_active(hpage) && !page_count(hpage)) {
4767 * Hwpoisoned hugepage isn't linked to activelist or freelist,
4768 * but dangling hpage->lru can trigger list-debug warnings
4769 * (this happens when we call unpoison_memory() on it),
4770 * so let it point to itself with list_del_init().
4772 list_del_init(&hpage->lru);
4773 set_page_refcounted(hpage);
4774 h->free_huge_pages--;
4775 h->free_huge_pages_node[nid]--;
4778 spin_unlock(&hugetlb_lock);
4783 bool isolate_huge_page(struct page *page, struct list_head *list)
4787 VM_BUG_ON_PAGE(!PageHead(page), page);
4788 spin_lock(&hugetlb_lock);
4789 if (!page_huge_active(page) || !get_page_unless_zero(page)) {
4793 clear_page_huge_active(page);
4794 list_move_tail(&page->lru, list);
4796 spin_unlock(&hugetlb_lock);
4800 void putback_active_hugepage(struct page *page)
4802 VM_BUG_ON_PAGE(!PageHead(page), page);
4803 spin_lock(&hugetlb_lock);
4804 set_page_huge_active(page);
4805 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
4806 spin_unlock(&hugetlb_lock);