Merge tag 'kvm-4.13-2' of git://git.kernel.org/pub/scm/virt/kvm/kvm

[karo-tx-linux.git] / drivers / iommu / iova.c

diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 5c88ba70e4e0fe92b282ebf1e8a1d0b2857677d4..246f14c83944c8c2bcff3bb25eafd8d13a262266 100644 (file)
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -22,6 +22,7 @@
 #include <linux/slab.h>
 #include <linux/smp.h>
 #include <linux/bitops.h>
+#include <linux/cpu.h>
 
 static bool iova_rcache_insert(struct iova_domain *iovad,
                               unsigned long pfn,
@@ -48,7 +49,7 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule,
        iovad->cached32_node = NULL;
        iovad->granule = granule;
        iovad->start_pfn = start_pfn;
-       iovad->dma_32bit_pfn = pfn_32bit;
+       iovad->dma_32bit_pfn = pfn_32bit + 1;
        init_iova_rcaches(iovad);
 }
 EXPORT_SYMBOL_GPL(init_iova_domain);
@@ -63,7 +64,7 @@ __get_cached_rbnode(struct iova_domain *iovad, unsigned long *limit_pfn)
                struct rb_node *prev_node = rb_prev(iovad->cached32_node);
                struct iova *curr_iova =
                        rb_entry(iovad->cached32_node, struct iova, node);
-               *limit_pfn = curr_iova->pfn_lo - 1;
+               *limit_pfn = curr_iova->pfn_lo;
                return prev_node;
        }
 }
@@ -135,7 +136,7 @@ iova_insert_rbtree(struct rb_root *root, struct iova *iova,
 static unsigned int
 iova_get_pad_size(unsigned int size, unsigned int limit_pfn)
 {
-       return (limit_pfn + 1 - size) & (__roundup_pow_of_two(size) - 1);
+       return (limit_pfn - size) & (__roundup_pow_of_two(size) - 1);
 }
 
 static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
@@ -155,18 +156,15 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
        while (curr) {
                struct iova *curr_iova = rb_entry(curr, struct iova, node);
 
-               if (limit_pfn < curr_iova->pfn_lo)
+               if (limit_pfn <= curr_iova->pfn_lo) {
                        goto move_left;
-               else if (limit_pfn < curr_iova->pfn_hi)
-                       goto adjust_limit_pfn;
-               else {
+               } else if (limit_pfn > curr_iova->pfn_hi) {
                        if (size_aligned)
                                pad_size = iova_get_pad_size(size, limit_pfn);
-                       if ((curr_iova->pfn_hi + size + pad_size) <= limit_pfn)
+                       if ((curr_iova->pfn_hi + size + pad_size) < limit_pfn)
                                break;  /* found a free slot */
                }
-adjust_limit_pfn:
-               limit_pfn = curr_iova->pfn_lo ? (curr_iova->pfn_lo - 1) : 0;
+               limit_pfn = curr_iova->pfn_lo;
 move_left:
                prev = curr;
                curr = rb_prev(curr);
@@ -182,7 +180,7 @@ move_left:
        }
 
        /* pfn_lo will point to size aligned address if size_aligned is set */
-       new->pfn_lo = limit_pfn - (size + pad_size) + 1;
+       new->pfn_lo = limit_pfn - (size + pad_size);
        new->pfn_hi = new->pfn_lo + size - 1;
 
        /* If we have 'prev', it's a valid place to start the insertion. */
@@ -269,7 +267,7 @@ alloc_iova(struct iova_domain *iovad, unsigned long size,
        if (!new_iova)
                return NULL;
 
-       ret = __alloc_and_insert_iova_range(iovad, size, limit_pfn,
+       ret = __alloc_and_insert_iova_range(iovad, size, limit_pfn + 1,
                        new_iova, size_aligned);
 
        if (ret) {
@@ -398,10 +396,8 @@ retry:
 
                /* Try replenishing IOVAs by flushing rcache. */
                flushed_rcache = true;
-               preempt_disable();
                for_each_online_cpu(cpu)
                        free_cpu_cached_iovas(cpu, iovad);
-               preempt_enable();
                goto retry;
        }
 
@@ -729,7 +725,7 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
        bool can_insert = false;
        unsigned long flags;
 
-       cpu_rcache = get_cpu_ptr(rcache->cpu_rcaches);
+       cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
        spin_lock_irqsave(&cpu_rcache->lock, flags);
 
        if (!iova_magazine_full(cpu_rcache->loaded)) {
@@ -759,7 +755,6 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
                iova_magazine_push(cpu_rcache->loaded, iova_pfn);
 
        spin_unlock_irqrestore(&cpu_rcache->lock, flags);
-       put_cpu_ptr(rcache->cpu_rcaches);
 
        if (mag_to_free) {
                iova_magazine_free_pfns(mag_to_free, iovad);
@@ -793,7 +788,7 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
        bool has_pfn = false;
        unsigned long flags;
 
-       cpu_rcache = get_cpu_ptr(rcache->cpu_rcaches);
+       cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
        spin_lock_irqsave(&cpu_rcache->lock, flags);
 
        if (!iova_magazine_empty(cpu_rcache->loaded)) {
@@ -815,7 +810,6 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
                iova_pfn = iova_magazine_pop(cpu_rcache->loaded, limit_pfn);
 
        spin_unlock_irqrestore(&cpu_rcache->lock, flags);
-       put_cpu_ptr(rcache->cpu_rcaches);
 
        return iova_pfn;
 }