mm: migrate: support non-lru movable page migration

[karo-tx-linux.git] / mm / compaction.c

diff --git a/mm/compaction.c b/mm/compaction.c
index 7bc04778f84dd6e2b68e4fadec74530f4bc64d57..fe95d8d021c30fe067b47ae7544d3070aac47ac8 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -81,6 +81,44 @@ static inline bool migrate_async_suitable(int migratetype)
 
 #ifdef CONFIG_COMPACTION
 
+int PageMovable(struct page *page)
+{
+       struct address_space *mapping;
+
+       VM_BUG_ON_PAGE(!PageLocked(page), page);
+       if (!__PageMovable(page))
+               return 0;
+
+       mapping = page_mapping(page);
+       if (mapping && mapping->a_ops && mapping->a_ops->isolate_page)
+               return 1;
+
+       return 0;
+}
+EXPORT_SYMBOL(PageMovable);
+
+void __SetPageMovable(struct page *page, struct address_space *mapping)
+{
+       VM_BUG_ON_PAGE(!PageLocked(page), page);
+       VM_BUG_ON_PAGE((unsigned long)mapping & PAGE_MAPPING_MOVABLE, page);
+       page->mapping = (void *)((unsigned long)mapping | PAGE_MAPPING_MOVABLE);
+}
+EXPORT_SYMBOL(__SetPageMovable);
+
+void __ClearPageMovable(struct page *page)
+{
+       VM_BUG_ON_PAGE(!PageLocked(page), page);
+       VM_BUG_ON_PAGE(!PageMovable(page), page);
+       /*
+        * Clear registered address_space val with keeping PAGE_MAPPING_MOVABLE
+        * flag so that VM can catch up released page by driver after isolation.
+        * With it, VM migration doesn't try to put it back.
+        */
+       page->mapping = (void *)((unsigned long)page->mapping &
+                               PAGE_MAPPING_MOVABLE);
+}
+EXPORT_SYMBOL(__ClearPageMovable);
+
 /* Do not skip compaction more than 64 times */
 #define COMPACT_MAX_DEFER_SHIFT 6
 
@@ -670,7 +708,6 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
 
        /* Time to isolate some pages for migration */
        for (; low_pfn < end_pfn; low_pfn++) {
-               bool is_lru;
 
                if (skip_on_failure && low_pfn >= next_skip_pfn) {
                        /*
@@ -732,21 +769,6 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                        continue;
                }
 
-               /*
-                * Check may be lockless but that's ok as we recheck later.
-                * It's possible to migrate LRU pages and balloon pages
-                * Skip any other type of page
-                */
-               is_lru = PageLRU(page);
-               if (!is_lru) {
-                       if (unlikely(balloon_page_movable(page))) {
-                               if (balloon_page_isolate(page)) {
-                                       /* Successfully isolated */
-                                       goto isolate_success;
-                               }
-                       }
-               }
-
                /*
                 * Regardless of being on LRU, compound pages such as THP and
                 * hugetlbfs are not to be compacted. We can potentially save
@@ -763,8 +785,37 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                        goto isolate_fail;
                }
 
-               if (!is_lru)
+               /*
+                * Check may be lockless but that's ok as we recheck later.
+                * It's possible to migrate LRU and non-lru movable pages.
+                * Skip any other type of page
+                */
+               if (!PageLRU(page)) {
+                       if (unlikely(balloon_page_movable(page))) {
+                               if (balloon_page_isolate(page)) {
+                                       /* Successfully isolated */
+                                       goto isolate_success;
+                               }
+                       }
+
+                       /*
+                        * __PageMovable can return false positive so we need
+                        * to verify it under page_lock.
+                        */
+                       if (unlikely(__PageMovable(page)) &&
+                                       !PageIsolated(page)) {
+                               if (locked) {
+                                       spin_unlock_irqrestore(&zone->lru_lock,
+                                                                       flags);
+                                       locked = false;
+                               }
+
+                               if (isolate_movable_page(page, isolate_mode))
+                                       goto isolate_success;
+                       }
+
                        goto isolate_fail;
+               }
 
                /*
                 * Migration will fail if an anonymous page is pinned in memory,