mm, page_alloc: inline pageblock lookup in page free fast paths

The function call overhead of get_pfnblock_flags_mask() is measurable in
the page free paths.  This patch uses an inlined version that is faster.

Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 4b28d2f8125e2ff3117695a083b1b7c603d1696c..c60db2096fd82fbaf92599df32770b6a095442d8 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -85,13 +85,6 @@ extern int page_group_by_mobility_disabled;
        get_pfnblock_flags_mask(page, page_to_pfn(page),                \
                        PB_migrate_end, MIGRATETYPE_MASK)
 
-static inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
-{
-       BUILD_BUG_ON(PB_migrate_end - PB_migrate != 2);
-       return get_pfnblock_flags_mask(page, pfn, PB_migrate_end,
-                                       MIGRATETYPE_MASK);
-}
-
 struct free_area {
        struct list_head        free_list[MIGRATE_TYPES];
        unsigned long           nr_free;

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 822ce86fc8837ba7784097d11c4e9b1c2799e24a..bdf7a13311b5a5b09b0aaa4ace0845d13ec993f4 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -352,6 +352,106 @@ static inline bool update_defer_init(pg_data_t *pgdat,
 }
 #endif
 
+/* Return a pointer to the bitmap storing bits affecting a block of pages */
+static inline unsigned long *get_pageblock_bitmap(struct page *page,
+                                                       unsigned long pfn)
+{
+#ifdef CONFIG_SPARSEMEM
+       return __pfn_to_section(pfn)->pageblock_flags;
+#else
+       return page_zone(page)->pageblock_flags;
+#endif /* CONFIG_SPARSEMEM */
+}
+
+static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
+{
+#ifdef CONFIG_SPARSEMEM
+       pfn &= (PAGES_PER_SECTION-1);
+       return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
+#else
+       pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
+       return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
+#endif /* CONFIG_SPARSEMEM */
+}
+
+/**
+ * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
+ * @page: The page within the block of interest
+ * @pfn: The target page frame number
+ * @end_bitidx: The last bit of interest to retrieve
+ * @mask: mask of bits that the caller is interested in
+ *
+ * Return: pageblock_bits flags
+ */
+static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page,
+                                       unsigned long pfn,
+                                       unsigned long end_bitidx,
+                                       unsigned long mask)
+{
+       unsigned long *bitmap;
+       unsigned long bitidx, word_bitidx;
+       unsigned long word;
+
+       bitmap = get_pageblock_bitmap(page, pfn);
+       bitidx = pfn_to_bitidx(page, pfn);
+       word_bitidx = bitidx / BITS_PER_LONG;
+       bitidx &= (BITS_PER_LONG-1);
+
+       word = bitmap[word_bitidx];
+       bitidx += end_bitidx;
+       return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
+}
+
+unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
+                                       unsigned long end_bitidx,
+                                       unsigned long mask)
+{
+       return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask);
+}
+
+static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
+{
+       return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK);
+}
+
+/**
+ * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
+ * @page: The page within the block of interest
+ * @flags: The flags to set
+ * @pfn: The target page frame number
+ * @end_bitidx: The last bit of interest
+ * @mask: mask of bits that the caller is interested in
+ */
+void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
+                                       unsigned long pfn,
+                                       unsigned long end_bitidx,
+                                       unsigned long mask)
+{
+       unsigned long *bitmap;
+       unsigned long bitidx, word_bitidx;
+       unsigned long old_word, word;
+
+       BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
+
+       bitmap = get_pageblock_bitmap(page, pfn);
+       bitidx = pfn_to_bitidx(page, pfn);
+       word_bitidx = bitidx / BITS_PER_LONG;
+       bitidx &= (BITS_PER_LONG-1);
+
+       VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
+
+       bitidx += end_bitidx;
+       mask <<= (BITS_PER_LONG - bitidx - 1);
+       flags <<= (BITS_PER_LONG - bitidx - 1);
+
+       word = READ_ONCE(bitmap[word_bitidx]);
+       for (;;) {
+               old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
+               if (word == old_word)
+                       break;
+               word = old_word;
+       }
+}
 
 void set_pageblock_migratetype(struct page *page, int migratetype)
 {
@@ -6831,94 +6931,6 @@ void *__init alloc_large_system_hash(const char *tablename,
        return table;
 }
 
-/* Return a pointer to the bitmap storing bits affecting a block of pages */
-static inline unsigned long *get_pageblock_bitmap(struct page *page,
-                                                       unsigned long pfn)
-{
-#ifdef CONFIG_SPARSEMEM
-       return __pfn_to_section(pfn)->pageblock_flags;
-#else
-       return page_zone(page)->pageblock_flags;
-#endif /* CONFIG_SPARSEMEM */
-}
-
-static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
-{
-#ifdef CONFIG_SPARSEMEM
-       pfn &= (PAGES_PER_SECTION-1);
-       return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
-#else
-       pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
-       return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
-#endif /* CONFIG_SPARSEMEM */
-}
-
-/**
- * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
- * @page: The page within the block of interest
- * @pfn: The target page frame number
- * @end_bitidx: The last bit of interest to retrieve
- * @mask: mask of bits that the caller is interested in
- *
- * Return: pageblock_bits flags
- */
-unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
-                                       unsigned long end_bitidx,
-                                       unsigned long mask)
-{
-       unsigned long *bitmap;
-       unsigned long bitidx, word_bitidx;
-       unsigned long word;
-
-       bitmap = get_pageblock_bitmap(page, pfn);
-       bitidx = pfn_to_bitidx(page, pfn);
-       word_bitidx = bitidx / BITS_PER_LONG;
-       bitidx &= (BITS_PER_LONG-1);
-
-       word = bitmap[word_bitidx];
-       bitidx += end_bitidx;
-       return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
-}
-
-/**
- * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
- * @page: The page within the block of interest
- * @flags: The flags to set
- * @pfn: The target page frame number
- * @end_bitidx: The last bit of interest
- * @mask: mask of bits that the caller is interested in
- */
-void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
-                                       unsigned long pfn,
-                                       unsigned long end_bitidx,
-                                       unsigned long mask)
-{
-       unsigned long *bitmap;
-       unsigned long bitidx, word_bitidx;
-       unsigned long old_word, word;
-
-       BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
-
-       bitmap = get_pageblock_bitmap(page, pfn);
-       bitidx = pfn_to_bitidx(page, pfn);
-       word_bitidx = bitidx / BITS_PER_LONG;
-       bitidx &= (BITS_PER_LONG-1);
-
-       VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
-
-       bitidx += end_bitidx;
-       mask <<= (BITS_PER_LONG - bitidx - 1);
-       flags <<= (BITS_PER_LONG - bitidx - 1);
-
-       word = READ_ONCE(bitmap[word_bitidx]);
-       for (;;) {
-               old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
-               if (word == old_word)
-                       break;
-               word = old_word;
-       }
-}
-
 /*
  * This function checks whether pageblock includes unmovable pages or not.
  * If @count is not zero, it is okay to include less @count unmovable pages

diff --git a/mm/page_owner.c b/mm/page_owner.c
index 438768c092acbf89faa19338688b947d0b0f81c3..792b56da13d8564f4f6fbab68dcf46297bd13c30 100644 (file)
--- a/mm/page_owner.c
+++ b/mm/page_owner.c
@@ -143,7 +143,7 @@ print_page_owner(char __user *buf, size_t count, unsigned long pfn,
                goto err;
 
        /* Print information relevant to grouping pages by mobility */
-       pageblock_mt = get_pfnblock_migratetype(page, pfn);
+       pageblock_mt = get_pageblock_migratetype(page);
        page_mt  = gfpflags_to_migratetype(page_ext->gfp_mask);
        ret += snprintf(kbuf + ret, count - ret,
                        "PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n",

diff --git a/mm/vmstat.c b/mm/vmstat.c
index f1a73bfb77b529048e08adc4bbe19e262afe03d1..5b72a8ad281353223fe678e0ae8df354453303ca 100644 (file)
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1041,7 +1041,7 @@ static void pagetypeinfo_showmixedcount_print(struct seq_file *m,
                block_end_pfn = min(block_end_pfn, end_pfn);
 
                page = pfn_to_page(pfn);
-               pageblock_mt = get_pfnblock_migratetype(page, pfn);
+               pageblock_mt = get_pageblock_migratetype(page);
 
                for (; pfn < block_end_pfn; pfn++) {
                        if (!pfn_valid_within(pfn))