Merge branch 'cpuidle' into release

[karo-tx-linux.git] / arch / arm64 / include / asm / tlbflush.h

diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index 934815d45eda0f180aa3bc5d5a6d043d6832cb8d..7bd2da021658ad765aa6bc93374f443ad0d7b594 100644 (file)
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -87,27 +87,56 @@ static inline void flush_tlb_page(struct vm_area_struct *vma,
                ((unsigned long)ASID(vma->vm_mm) << 48);
 
        dsb(ishst);
-       asm("tlbi       vae1is, %0" : : "r" (addr));
+       asm("tlbi       vale1is, %0" : : "r" (addr));
        dsb(ish);
 }
 
+/*
+ * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
+ * necessarily a performance improvement.
+ */
+#define MAX_TLB_RANGE  (1024UL << PAGE_SHIFT)
+
 static inline void __flush_tlb_range(struct vm_area_struct *vma,
-                                    unsigned long start, unsigned long end)
+                                    unsigned long start, unsigned long end,
+                                    bool last_level)
 {
        unsigned long asid = (unsigned long)ASID(vma->vm_mm) << 48;
        unsigned long addr;
+
+       if ((end - start) > MAX_TLB_RANGE) {
+               flush_tlb_mm(vma->vm_mm);
+               return;
+       }
+
        start = asid | (start >> 12);
        end = asid | (end >> 12);
 
        dsb(ishst);
-       for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
-               asm("tlbi vae1is, %0" : : "r"(addr));
+       for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) {
+               if (last_level)
+                       asm("tlbi vale1is, %0" : : "r"(addr));
+               else
+                       asm("tlbi vae1is, %0" : : "r"(addr));
+       }
        dsb(ish);
 }
 
-static inline void __flush_tlb_kernel_range(unsigned long start, unsigned long end)
+static inline void flush_tlb_range(struct vm_area_struct *vma,
+                                  unsigned long start, unsigned long end)
+{
+       __flush_tlb_range(vma, start, end, false);
+}
+
+static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 {
        unsigned long addr;
+
+       if ((end - start) > MAX_TLB_RANGE) {
+               flush_tlb_all();
+               return;
+       }
+
        start >>= 12;
        end >>= 12;
 
@@ -118,29 +147,6 @@ static inline void __flush_tlb_kernel_range(unsigned long start, unsigned long e
        isb();
 }
 
-/*
- * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
- * necessarily a performance improvement.
- */
-#define MAX_TLB_RANGE  (1024UL << PAGE_SHIFT)
-
-static inline void flush_tlb_range(struct vm_area_struct *vma,
-                                  unsigned long start, unsigned long end)
-{
-       if ((end - start) <= MAX_TLB_RANGE)
-               __flush_tlb_range(vma, start, end);
-       else
-               flush_tlb_mm(vma->vm_mm);
-}
-
-static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
-{
-       if ((end - start) <= MAX_TLB_RANGE)
-               __flush_tlb_kernel_range(start, end);
-       else
-               flush_tlb_all();
-}
-
 /*
  * Used to invalidate the TLB (walk caches) corresponding to intermediate page
  * table levels (pgd/pud/pmd).
@@ -154,20 +160,6 @@ static inline void __flush_tlb_pgtable(struct mm_struct *mm,
        asm("tlbi       vae1is, %0" : : "r" (addr));
        dsb(ish);
 }
-/*
- * On AArch64, the cache coherency is handled via the set_pte_at() function.
- */
-static inline void update_mmu_cache(struct vm_area_struct *vma,
-                                   unsigned long addr, pte_t *ptep)
-{
-       /*
-        * set_pte() does not have a DSB for user mappings, so make sure that
-        * the page table write is visible.
-        */
-       dsb(ishst);
-}
-
-#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
 
 #endif