nvme-pci: quiesce/unquiesce admin_q instead of start/stop its hw queues

[karo-tx-linux.git] / fs / dax.c

diff --git a/fs/dax.c b/fs/dax.c
index 123d9903c77d859c1ecfa79022ef66c68ce294c2..2a6889b3585f068c73091d8895639b7e941d702a 100644 (file)
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -964,12 +964,12 @@ int __dax_zero_page_range(struct block_device *bdev,
                void *kaddr;
                pfn_t pfn;
 
-               rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
+               rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);
                if (rc)
                        return rc;
 
                id = dax_read_lock();
-               rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr,
+               rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr,
                                &pfn);
                if (rc < 0) {
                        dax_read_unlock(id);
@@ -1148,6 +1148,23 @@ static int dax_iomap_pte_fault(struct vm_fault *vmf,
        if ((vmf->flags & FAULT_FLAG_WRITE) && !vmf->cow_page)
                flags |= IOMAP_WRITE;
 
+       entry = grab_mapping_entry(mapping, vmf->pgoff, 0);
+       if (IS_ERR(entry)) {
+               vmf_ret = dax_fault_return(PTR_ERR(entry));
+               goto out;
+       }
+
+       /*
+        * It is possible, particularly with mixed reads & writes to private
+        * mappings, that we have raced with a PMD fault that overlaps with
+        * the PTE we need to set up.  If so just return and the fault will be
+        * retried.
+        */
+       if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) {
+               vmf_ret = VM_FAULT_NOPAGE;
+               goto unlock_entry;
+       }
+
        /*
         * Note that we don't bother to use iomap_apply here: DAX required
         * the file system block size to be equal the page size, which means
@@ -1156,17 +1173,11 @@ static int dax_iomap_pte_fault(struct vm_fault *vmf,
        error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap);
        if (error) {
                vmf_ret = dax_fault_return(error);
-               goto out;
+               goto unlock_entry;
        }
        if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
-               vmf_ret = dax_fault_return(-EIO);       /* fs corruption? */
-               goto finish_iomap;
-       }
-
-       entry = grab_mapping_entry(mapping, vmf->pgoff, 0);
-       if (IS_ERR(entry)) {
-               vmf_ret = dax_fault_return(PTR_ERR(entry));
-               goto finish_iomap;
+               error = -EIO;   /* fs corruption? */
+               goto error_finish_iomap;
        }
 
        sector = dax_iomap_sector(&iomap, pos);
@@ -1188,13 +1199,13 @@ static int dax_iomap_pte_fault(struct vm_fault *vmf,
                }
 
                if (error)
-                       goto error_unlock_entry;
+                       goto error_finish_iomap;
 
                __SetPageUptodate(vmf->cow_page);
                vmf_ret = finish_fault(vmf);
                if (!vmf_ret)
                        vmf_ret = VM_FAULT_DONE_COW;
-               goto unlock_entry;
+               goto finish_iomap;
        }
 
        switch (iomap.type) {
@@ -1214,7 +1225,7 @@ static int dax_iomap_pte_fault(struct vm_fault *vmf,
        case IOMAP_HOLE:
                if (!(vmf->flags & FAULT_FLAG_WRITE)) {
                        vmf_ret = dax_load_hole(mapping, &entry, vmf);
-                       goto unlock_entry;
+                       goto finish_iomap;
                }
                /*FALLTHRU*/
        default:
@@ -1223,10 +1234,8 @@ static int dax_iomap_pte_fault(struct vm_fault *vmf,
                break;
        }
 
- error_unlock_entry:
+ error_finish_iomap:
        vmf_ret = dax_fault_return(error) | major;
- unlock_entry:
-       put_locked_mapping_entry(mapping, vmf->pgoff, entry);
  finish_iomap:
        if (ops->iomap_end) {
                int copied = PAGE_SIZE;
@@ -1241,7 +1250,9 @@ static int dax_iomap_pte_fault(struct vm_fault *vmf,
                 */
                ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap);
        }
-out:
+ unlock_entry:
+       put_locked_mapping_entry(mapping, vmf->pgoff, entry);
+ out:
        trace_dax_pte_fault_done(inode, vmf, vmf_ret);
        return vmf_ret;
 }
@@ -1387,6 +1398,28 @@ static int dax_iomap_pmd_fault(struct vm_fault *vmf,
        if ((pgoff | PG_PMD_COLOUR) > max_pgoff)
                goto fallback;
 
+       /*
+        * grab_mapping_entry() will make sure we get a 2M empty entry, a DAX
+        * PMD or a HZP entry.  If it can't (because a 4k page is already in
+        * the tree, for instance), it will return -EEXIST and we just fall
+        * back to 4k entries.
+        */
+       entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD);
+       if (IS_ERR(entry))
+               goto fallback;
+
+       /*
+        * It is possible, particularly with mixed reads & writes to private
+        * mappings, that we have raced with a PTE fault that overlaps with
+        * the PMD we need to set up.  If so just return and the fault will be
+        * retried.
+        */
+       if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) &&
+                       !pmd_devmap(*vmf->pmd)) {
+               result = 0;
+               goto unlock_entry;
+       }
+
        /*
         * Note that we don't use iomap_apply here.  We aren't doing I/O, only
         * setting up a mapping, so really we're using iomap_begin() as a way
@@ -1395,21 +1428,11 @@ static int dax_iomap_pmd_fault(struct vm_fault *vmf,
        pos = (loff_t)pgoff << PAGE_SHIFT;
        error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap);
        if (error)
-               goto fallback;
+               goto unlock_entry;
 
        if (iomap.offset + iomap.length < pos + PMD_SIZE)
                goto finish_iomap;
 
-       /*
-        * grab_mapping_entry() will make sure we get a 2M empty entry, a DAX
-        * PMD or a HZP entry.  If it can't (because a 4k page is already in
-        * the tree, for instance), it will return -EEXIST and we just fall
-        * back to 4k entries.
-        */
-       entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD);
-       if (IS_ERR(entry))
-               goto finish_iomap;
-
        switch (iomap.type) {
        case IOMAP_MAPPED:
                result = dax_pmd_insert_mapping(vmf, &iomap, pos, &entry);
@@ -1417,7 +1440,7 @@ static int dax_iomap_pmd_fault(struct vm_fault *vmf,
        case IOMAP_UNWRITTEN:
        case IOMAP_HOLE:
                if (WARN_ON_ONCE(write))
-                       goto unlock_entry;
+                       break;
                result = dax_pmd_load_hole(vmf, &iomap, &entry);
                break;
        default:
@@ -1425,8 +1448,6 @@ static int dax_iomap_pmd_fault(struct vm_fault *vmf,
                break;
        }
 
- unlock_entry:
-       put_locked_mapping_entry(mapping, pgoff, entry);
  finish_iomap:
        if (ops->iomap_end) {
                int copied = PMD_SIZE;
@@ -1442,6 +1463,8 @@ static int dax_iomap_pmd_fault(struct vm_fault *vmf,
                ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags,
                                &iomap);
        }
+ unlock_entry:
+       put_locked_mapping_entry(mapping, pgoff, entry);
  fallback:
        if (result == VM_FAULT_FALLBACK) {
                split_huge_pmd(vma, vmf->pmd, vmf->address);