mm: fix page-faults detection in swap-token logic

After commit v2.6.36-5896-gd065bd8 "mm: retry page fault when blocking on
disk transfer" we usually wait in page-faults without mmap_sem held, so
all swap-token logic was broken, because it based on using
rwsem_is_locked(&mm->mmap_sem) as sign of in progress page-faults.

Add an atomic counter of in progress page-faults for mm to the mm_struct
with swap-token.

Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 26574c726121cb9faab6b6fd3af517c73840869f..a87eefbe6b072886fdbe142e1d8e54a06a039f8c 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -355,6 +355,7 @@ struct mm_struct {
        unsigned int faultstamp;
        unsigned int token_priority;
        unsigned int last_interval;
+       atomic_t active_swap_token;
 
        unsigned long flags; /* Must use atomic bitops to access the bits */
 

diff --git a/include/linux/swap.h b/include/linux/swap.h
index 50a55e2d58ec107c3efe207b61e2e683ae8798ee..a1f5d74f65d150d8ba711cf7c961d3bbb5606a45 100644 (file)
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -376,6 +376,26 @@ static inline void put_swap_token(struct mm_struct *mm)
                __put_swap_token(mm);
 }
 
+static inline bool has_active_swap_token(struct mm_struct *mm)
+{
+       return has_swap_token(mm) && atomic_read(&mm->active_swap_token);
+}
+
+static inline bool activate_swap_token(struct mm_struct *mm)
+{
+       if (has_swap_token(mm)) {
+               atomic_inc(&mm->active_swap_token);
+               return true;
+       }
+       return false;
+}
+
+static inline void deactivate_swap_token(struct mm_struct *mm, bool swap_token)
+{
+       if (swap_token)
+               atomic_dec(&mm->active_swap_token);
+}
+
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
 extern void
 mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout);
@@ -494,6 +514,20 @@ static inline int has_swap_token(struct mm_struct *mm)
        return 0;
 }
 
+static inline bool has_active_swap_token(struct mm_struct *mm)
+{
+       return false;
+}
+
+static inline bool activate_swap_token(struct mm_struct *mm)
+{
+       return false;
+}
+
+static inline void deactivate_swap_token(struct mm_struct *mm, bool swap_token)
+{
+}
+
 static inline void disable_swap_token(struct mem_cgroup *memcg)
 {
 }

diff --git a/kernel/fork.c b/kernel/fork.c
index ca9a3845ef3e1788f78708c6e554958ec04af5b4..f69154e160838f3c4760fc486a3758746ba193ad 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -804,6 +804,7 @@ struct mm_struct *dup_mm(struct task_struct *tsk)
        /* Initializing for Swap token stuff */
        mm->token_priority = 0;
        mm->last_interval = 0;
+       atomic_set(&mm->active_swap_token, 0);
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
        mm->pmd_huge_pte = NULL;

diff --git a/mm/memory.c b/mm/memory.c
index 6105f475fa8633edf5180792b2cf0c5288734f08..cdc4b4c1a8239d33929d0b477609721e3efe0881 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2892,6 +2892,7 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
        struct mem_cgroup *ptr;
        int exclusive = 0;
        int ret = 0;
+       bool swap_token;
 
        if (!pte_unmap_same(mm, pmd, page_table, orig_pte))
                goto out;
@@ -2940,7 +2941,12 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
                goto out_release;
        }
 
+       swap_token = activate_swap_token(mm);
+
        locked = lock_page_or_retry(page, mm, flags);
+
+       deactivate_swap_token(mm, swap_token);
+
        delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
        if (!locked) {
                ret |= VM_FAULT_RETRY;
@@ -3187,6 +3193,7 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
        struct vm_fault vmf;
        int ret;
        int page_mkwrite = 0;
+       bool swap_token;
 
        /*
         * If we do COW later, allocate page befor taking lock_page()
@@ -3208,6 +3215,8 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
        } else
                cow_page = NULL;
 
+       swap_token = activate_swap_token(mm);
+
        vmf.virtual_address = (void __user *)(address & PAGE_MASK);
        vmf.pgoff = pgoff;
        vmf.flags = flags;
@@ -3276,6 +3285,8 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
 
        }
 
+       deactivate_swap_token(mm, swap_token);
+
        page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
 
        /*
@@ -3347,9 +3358,11 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
        return ret;
 
 unwritable_page:
+       deactivate_swap_token(mm, swap_token);
        page_cache_release(page);
        return ret;
 uncharge_out:
+       deactivate_swap_token(mm, swap_token);
        /* fs's fault handler get error */
        if (cow_page) {
                mem_cgroup_uncharge_page(cow_page);

diff --git a/mm/rmap.c b/mm/rmap.c
index 5b5ad584ffb7dd7c9e00a885a018aaadba22f371..36d01a210135fa4c5bb67cd6c9477076112f56cf 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -757,8 +757,7 @@ int page_referenced_one(struct page *page, struct vm_area_struct *vma,
 
        /* Pretend the page is referenced if the task has the
           swap token and is in the middle of a page fault. */
-       if (mm != current->mm && has_swap_token(mm) &&
-                       rwsem_is_locked(&mm->mmap_sem))
+       if (mm != current->mm && has_active_swap_token(mm))
                referenced++;
 
        (*mapcount)--;