drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  *          Joerg Roedel <jroedel@suse.de>
  19  */
  20
  21 #define pr_fmt(fmt)     "DMAR: " fmt
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/export.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/memory.h>
  36 #include <linux/cpu.h>
  37 #include <linux/timer.h>
  38 #include <linux/io.h>
  39 #include <linux/iova.h>
  40 #include <linux/iommu.h>
  41 #include <linux/intel-iommu.h>
  42 #include <linux/syscore_ops.h>
  43 #include <linux/tboot.h>
  44 #include <linux/dmi.h>
  45 #include <linux/pci-ats.h>
  46 #include <linux/memblock.h>
  47 #include <linux/dma-contiguous.h>
  48 #include <linux/crash_dump.h>
  49 #include <asm/irq_remapping.h>
  50 #include <asm/cacheflush.h>
  51 #include <asm/iommu.h>
  52
  53 #include "irq_remapping.h"
  54
  55 #define ROOT_SIZE               VTD_PAGE_SIZE
  56 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  57
  58 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  59 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  60 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  61 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  62
  63 #define IOAPIC_RANGE_START      (0xfee00000)
  64 #define IOAPIC_RANGE_END        (0xfeefffff)
  65 #define IOVA_START_ADDR         (0x1000)
  66
  67 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  68
  69 #define MAX_AGAW_WIDTH 64
  70 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  71
  72 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  73 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  74
  75 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  76    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  77 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  78                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  79 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  80
  81 /* IO virtual address start page frame number */
  82 #define IOVA_START_PFN          (1)
  83
  84 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  85 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  86 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  87
  88 /* page table handling */
  89 #define LEVEL_STRIDE            (9)
  90 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  91
  92 /*
  93  * This bitmap is used to advertise the page sizes our hardware support
  94  * to the IOMMU core, which will then use this information to split
  95  * physically contiguous memory regions it is mapping into page sizes
  96  * that we support.
  97  *
  98  * Traditionally the IOMMU core just handed us the mappings directly,
  99  * after making sure the size is an order of a 4KiB page and that the
 100  * mapping has natural alignment.
 101  *
 102  * To retain this behavior, we currently advertise that we support
 103  * all page sizes that are an order of 4KiB.
 104  *
 105  * If at some point we'd like to utilize the IOMMU core's new behavior,
 106  * we could change this to advertise the real page sizes we support.
 107  */
 108 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 109
 110 static inline int agaw_to_level(int agaw)
 111 {
 112         return agaw + 2;
 113 }
 114
 115 static inline int agaw_to_width(int agaw)
 116 {
 117         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 118 }
 119
 120 static inline int width_to_agaw(int width)
 121 {
 122         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 123 }
 124
 125 static inline unsigned int level_to_offset_bits(int level)
 126 {
 127         return (level - 1) * LEVEL_STRIDE;
 128 }
 129
 130 static inline int pfn_level_offset(unsigned long pfn, int level)
 131 {
 132         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 133 }
 134
 135 static inline unsigned long level_mask(int level)
 136 {
 137         return -1UL << level_to_offset_bits(level);
 138 }
 139
 140 static inline unsigned long level_size(int level)
 141 {
 142         return 1UL << level_to_offset_bits(level);
 143 }
 144
 145 static inline unsigned long align_to_level(unsigned long pfn, int level)
 146 {
 147         return (pfn + level_size(level) - 1) & level_mask(level);
 148 }
 149
 150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 151 {
 152         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 153 }
 154
 155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 156    are never going to work. */
 157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 158 {
 159         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 160 }
 161
 162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 163 {
 164         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 165 }
 166 static inline unsigned long page_to_dma_pfn(struct page *pg)
 167 {
 168         return mm_to_dma_pfn(page_to_pfn(pg));
 169 }
 170 static inline unsigned long virt_to_dma_pfn(void *p)
 171 {
 172         return page_to_dma_pfn(virt_to_page(p));
 173 }
 174
 175 /* global iommu list, set NULL for ignored DMAR units */
 176 static struct intel_iommu **g_iommus;
 177
 178 static void __init check_tylersburg_isoch(void);
 179 static int rwbf_quirk;
 180
 181 /*
 182  * set to 1 to panic kernel if can't successfully enable VT-d
 183  * (used when kernel is launched w/ TXT)
 184  */
 185 static int force_on = 0;
 186
 187 /*
 188  * 0: Present
 189  * 1-11: Reserved
 190  * 12-63: Context Ptr (12 - (haw-1))
 191  * 64-127: Reserved
 192  */
 193 struct root_entry {
 194         u64     lo;
 195         u64     hi;
 196 };
 197 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 198
 199 /*
 200  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
 201  * if marked present.
 202  */
 203 static phys_addr_t root_entry_lctp(struct root_entry *re)
 204 {
 205         if (!(re->lo & 1))
 206                 return 0;
 207
 208         return re->lo & VTD_PAGE_MASK;
 209 }
 210
 211 /*
 212  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
 213  * if marked present.
 214  */
 215 static phys_addr_t root_entry_uctp(struct root_entry *re)
 216 {
 217         if (!(re->hi & 1))
 218                 return 0;
 219
 220         return re->hi & VTD_PAGE_MASK;
 221 }
 222 /*
 223  * low 64 bits:
 224  * 0: present
 225  * 1: fault processing disable
 226  * 2-3: translation type
 227  * 12-63: address space root
 228  * high 64 bits:
 229  * 0-2: address width
 230  * 3-6: aval
 231  * 8-23: domain id
 232  */
 233 struct context_entry {
 234         u64 lo;
 235         u64 hi;
 236 };
 237
 238 static inline void context_clear_pasid_enable(struct context_entry *context)
 239 {
 240         context->lo &= ~(1ULL << 11);
 241 }
 242
 243 static inline bool context_pasid_enabled(struct context_entry *context)
 244 {
 245         return !!(context->lo & (1ULL << 11));
 246 }
 247
 248 static inline void context_set_copied(struct context_entry *context)
 249 {
 250         context->hi |= (1ull << 3);
 251 }
 252
 253 static inline bool context_copied(struct context_entry *context)
 254 {
 255         return !!(context->hi & (1ULL << 3));
 256 }
 257
 258 static inline bool __context_present(struct context_entry *context)
 259 {
 260         return (context->lo & 1);
 261 }
 262
 263 static inline bool context_present(struct context_entry *context)
 264 {
 265         return context_pasid_enabled(context) ?
 266              __context_present(context) :
 267              __context_present(context) && !context_copied(context);
 268 }
 269
 270 static inline void context_set_present(struct context_entry *context)
 271 {
 272         context->lo |= 1;
 273 }
 274
 275 static inline void context_set_fault_enable(struct context_entry *context)
 276 {
 277         context->lo &= (((u64)-1) << 2) | 1;
 278 }
 279
 280 static inline void context_set_translation_type(struct context_entry *context,
 281                                                 unsigned long value)
 282 {
 283         context->lo &= (((u64)-1) << 4) | 3;
 284         context->lo |= (value & 3) << 2;
 285 }
 286
 287 static inline void context_set_address_root(struct context_entry *context,
 288                                             unsigned long value)
 289 {
 290         context->lo &= ~VTD_PAGE_MASK;
 291         context->lo |= value & VTD_PAGE_MASK;
 292 }
 293
 294 static inline void context_set_address_width(struct context_entry *context,
 295                                              unsigned long value)
 296 {
 297         context->hi |= value & 7;
 298 }
 299
 300 static inline void context_set_domain_id(struct context_entry *context,
 301                                          unsigned long value)
 302 {
 303         context->hi |= (value & ((1 << 16) - 1)) << 8;
 304 }
 305
 306 static inline int context_domain_id(struct context_entry *c)
 307 {
 308         return((c->hi >> 8) & 0xffff);
 309 }
 310
 311 static inline void context_clear_entry(struct context_entry *context)
 312 {
 313         context->lo = 0;
 314         context->hi = 0;
 315 }
 316
 317 /*
 318  * 0: readable
 319  * 1: writable
 320  * 2-6: reserved
 321  * 7: super page
 322  * 8-10: available
 323  * 11: snoop behavior
 324  * 12-63: Host physcial address
 325  */
 326 struct dma_pte {
 327         u64 val;
 328 };
 329
 330 static inline void dma_clear_pte(struct dma_pte *pte)
 331 {
 332         pte->val = 0;
 333 }
 334
 335 static inline u64 dma_pte_addr(struct dma_pte *pte)
 336 {
 337 #ifdef CONFIG_64BIT
 338         return pte->val & VTD_PAGE_MASK;
 339 #else
 340         /* Must have a full atomic 64-bit read */
 341         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 342 #endif
 343 }
 344
 345 static inline bool dma_pte_present(struct dma_pte *pte)
 346 {
 347         return (pte->val & 3) != 0;
 348 }
 349
 350 static inline bool dma_pte_superpage(struct dma_pte *pte)
 351 {
 352         return (pte->val & DMA_PTE_LARGE_PAGE);
 353 }
 354
 355 static inline int first_pte_in_page(struct dma_pte *pte)
 356 {
 357         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 358 }
 359
 360 /*
 361  * This domain is a statically identity mapping domain.
 362  *      1. This domain creats a static 1:1 mapping to all usable memory.
 363  *      2. It maps to each iommu if successful.
 364  *      3. Each iommu mapps to this domain if successful.
 365  */
 366 static struct dmar_domain *si_domain;
 367 static int hw_pass_through = 1;
 368
 369 /*
 370  * Domain represents a virtual machine, more than one devices
 371  * across iommus may be owned in one domain, e.g. kvm guest.
 372  */
 373 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
 374
 375 /* si_domain contains mulitple devices */
 376 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
 377
 378 #define for_each_domain_iommu(idx, domain)                      \
 379         for (idx = 0; idx < g_num_of_iommus; idx++)             \
 380                 if (domain->iommu_refcnt[idx])
 381
 382 struct dmar_domain {
 383         int     nid;                    /* node id */
 384
 385         unsigned        iommu_refcnt[DMAR_UNITS_SUPPORTED];
 386                                         /* Refcount of devices per iommu */
 387
 388
 389         u16             iommu_did[DMAR_UNITS_SUPPORTED];
 390                                         /* Domain ids per IOMMU. Use u16 since
 391                                          * domain ids are 16 bit wide according
 392                                          * to VT-d spec, section 9.3 */
 393
 394         bool has_iotlb_device;
 395         struct list_head devices;       /* all devices' list */
 396         struct iova_domain iovad;       /* iova's that belong to this domain */
 397
 398         struct dma_pte  *pgd;           /* virtual address */
 399         int             gaw;            /* max guest address width */
 400
 401         /* adjusted guest address width, 0 is level 2 30-bit */
 402         int             agaw;
 403
 404         int             flags;          /* flags to find out type of domain */
 405
 406         int             iommu_coherency;/* indicate coherency of iommu access */
 407         int             iommu_snooping; /* indicate snooping control feature*/
 408         int             iommu_count;    /* reference count of iommu */
 409         int             iommu_superpage;/* Level of superpages supported:
 410                                            0 == 4KiB (no superpages), 1 == 2MiB,
 411                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 412         u64             max_addr;       /* maximum mapped address */
 413
 414         struct iommu_domain domain;     /* generic domain data structure for
 415                                            iommu core */
 416 };
 417
 418 /* PCI domain-device relationship */
 419 struct device_domain_info {
 420         struct list_head link;  /* link to domain siblings */
 421         struct list_head global; /* link to global list */
 422         u8 bus;                 /* PCI bus number */
 423         u8 devfn;               /* PCI devfn number */
 424         u8 pasid_supported:3;
 425         u8 pasid_enabled:1;
 426         u8 pri_supported:1;
 427         u8 pri_enabled:1;
 428         u8 ats_supported:1;
 429         u8 ats_enabled:1;
 430         u8 ats_qdep;
 431         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
 432         struct intel_iommu *iommu; /* IOMMU used by this device */
 433         struct dmar_domain *domain; /* pointer to domain */
 434 };
 435
 436 struct dmar_rmrr_unit {
 437         struct list_head list;          /* list of rmrr units   */
 438         struct acpi_dmar_header *hdr;   /* ACPI header          */
 439         u64     base_address;           /* reserved base address*/
 440         u64     end_address;            /* reserved end address */
 441         struct dmar_dev_scope *devices; /* target devices */
 442         int     devices_cnt;            /* target device count */
 443 };
 444
 445 struct dmar_atsr_unit {
 446         struct list_head list;          /* list of ATSR units */
 447         struct acpi_dmar_header *hdr;   /* ACPI header */
 448         struct dmar_dev_scope *devices; /* target devices */
 449         int devices_cnt;                /* target device count */
 450         u8 include_all:1;               /* include all ports */
 451 };
 452
 453 static LIST_HEAD(dmar_atsr_units);
 454 static LIST_HEAD(dmar_rmrr_units);
 455
 456 #define for_each_rmrr_units(rmrr) \
 457         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 458
 459 static void flush_unmaps_timeout(unsigned long data);
 460
 461 struct deferred_flush_entry {
 462         unsigned long iova_pfn;
 463         unsigned long nrpages;
 464         struct dmar_domain *domain;
 465         struct page *freelist;
 466 };
 467
 468 #define HIGH_WATER_MARK 250
 469 struct deferred_flush_table {
 470         int next;
 471         struct deferred_flush_entry entries[HIGH_WATER_MARK];
 472 };
 473
 474 struct deferred_flush_data {
 475         spinlock_t lock;
 476         int timer_on;
 477         struct timer_list timer;
 478         long size;
 479         struct deferred_flush_table *tables;
 480 };
 481
 482 DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
 483
 484 /* bitmap for indexing intel_iommus */
 485 static int g_num_of_iommus;
 486
 487 static void domain_exit(struct dmar_domain *domain);
 488 static void domain_remove_dev_info(struct dmar_domain *domain);
 489 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
 490                                      struct device *dev);
 491 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
 492 static void domain_context_clear(struct intel_iommu *iommu,
 493                                  struct device *dev);
 494 static int domain_detach_iommu(struct dmar_domain *domain,
 495                                struct intel_iommu *iommu);
 496
 497 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 498 int dmar_disabled = 0;
 499 #else
 500 int dmar_disabled = 1;
 501 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 502
 503 int intel_iommu_enabled = 0;
 504 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 505
 506 static int dmar_map_gfx = 1;
 507 static int dmar_forcedac;
 508 static int intel_iommu_strict;
 509 static int intel_iommu_superpage = 1;
 510 static int intel_iommu_ecs = 1;
 511 static int intel_iommu_pasid28;
 512 static int iommu_identity_mapping;
 513
 514 #define IDENTMAP_ALL            1
 515 #define IDENTMAP_GFX            2
 516 #define IDENTMAP_AZALIA         4
 517
 518 /* Broadwell and Skylake have broken ECS support — normal so-called "second
 519  * level" translation of DMA requests-without-PASID doesn't actually happen
 520  * unless you also set the NESTE bit in an extended context-entry. Which of
 521  * course means that SVM doesn't work because it's trying to do nested
 522  * translation of the physical addresses it finds in the process page tables,
 523  * through the IOVA->phys mapping found in the "second level" page tables.
 524  *
 525  * The VT-d specification was retroactively changed to change the definition
 526  * of the capability bits and pretend that Broadwell/Skylake never happened...
 527  * but unfortunately the wrong bit was changed. It's ECS which is broken, but
 528  * for some reason it was the PASID capability bit which was redefined (from
 529  * bit 28 on BDW/SKL to bit 40 in future).
 530  *
 531  * So our test for ECS needs to eschew those implementations which set the old
 532  * PASID capabiity bit 28, since those are the ones on which ECS is broken.
 533  * Unless we are working around the 'pasid28' limitations, that is, by putting
 534  * the device into passthrough mode for normal DMA and thus masking the bug.
 535  */
 536 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
 537                             (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
 538 /* PASID support is thus enabled if ECS is enabled and *either* of the old
 539  * or new capability bits are set. */
 540 #define pasid_enabled(iommu) (ecs_enabled(iommu) &&                     \
 541                               (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
 542
 543 int intel_iommu_gfx_mapped;
 544 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 545
 546 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 547 static DEFINE_SPINLOCK(device_domain_lock);
 548 static LIST_HEAD(device_domain_list);
 549
 550 static const struct iommu_ops intel_iommu_ops;
 551
 552 static bool translation_pre_enabled(struct intel_iommu *iommu)
 553 {
 554         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 555 }
 556
 557 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 558 {
 559         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 560 }
 561
 562 static void init_translation_status(struct intel_iommu *iommu)
 563 {
 564         u32 gsts;
 565
 566         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 567         if (gsts & DMA_GSTS_TES)
 568                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 569 }
 570
 571 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 572 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 573 {
 574         return container_of(dom, struct dmar_domain, domain);
 575 }
 576
 577 static int __init intel_iommu_setup(char *str)
 578 {
 579         if (!str)
 580                 return -EINVAL;
 581         while (*str) {
 582                 if (!strncmp(str, "on", 2)) {
 583                         dmar_disabled = 0;
 584                         pr_info("IOMMU enabled\n");
 585                 } else if (!strncmp(str, "off", 3)) {
 586                         dmar_disabled = 1;
 587                         pr_info("IOMMU disabled\n");
 588                 } else if (!strncmp(str, "igfx_off", 8)) {
 589                         dmar_map_gfx = 0;
 590                         pr_info("Disable GFX device mapping\n");
 591                 } else if (!strncmp(str, "forcedac", 8)) {
 592                         pr_info("Forcing DAC for PCI devices\n");
 593                         dmar_forcedac = 1;
 594                 } else if (!strncmp(str, "strict", 6)) {
 595                         pr_info("Disable batched IOTLB flush\n");
 596                         intel_iommu_strict = 1;
 597                 } else if (!strncmp(str, "sp_off", 6)) {
 598                         pr_info("Disable supported super page\n");
 599                         intel_iommu_superpage = 0;
 600                 } else if (!strncmp(str, "ecs_off", 7)) {
 601                         printk(KERN_INFO
 602                                 "Intel-IOMMU: disable extended context table support\n");
 603                         intel_iommu_ecs = 0;
 604                 } else if (!strncmp(str, "pasid28", 7)) {
 605                         printk(KERN_INFO
 606                                 "Intel-IOMMU: enable pre-production PASID support\n");
 607                         intel_iommu_pasid28 = 1;
 608                         iommu_identity_mapping |= IDENTMAP_GFX;
 609                 }
 610
 611                 str += strcspn(str, ",");
 612                 while (*str == ',')
 613                         str++;
 614         }
 615         return 0;
 616 }
 617 __setup("intel_iommu=", intel_iommu_setup);
 618
 619 static struct kmem_cache *iommu_domain_cache;
 620 static struct kmem_cache *iommu_devinfo_cache;
 621
 622 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
 623 {
 624         struct dmar_domain **domains;
 625         int idx = did >> 8;
 626
 627         domains = iommu->domains[idx];
 628         if (!domains)
 629                 return NULL;
 630
 631         return domains[did & 0xff];
 632 }
 633
 634 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
 635                              struct dmar_domain *domain)
 636 {
 637         struct dmar_domain **domains;
 638         int idx = did >> 8;
 639
 640         if (!iommu->domains[idx]) {
 641                 size_t size = 256 * sizeof(struct dmar_domain *);
 642                 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
 643         }
 644
 645         domains = iommu->domains[idx];
 646         if (WARN_ON(!domains))
 647                 return;
 648         else
 649                 domains[did & 0xff] = domain;
 650 }
 651
 652 static inline void *alloc_pgtable_page(int node)
 653 {
 654         struct page *page;
 655         void *vaddr = NULL;
 656
 657         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 658         if (page)
 659                 vaddr = page_address(page);
 660         return vaddr;
 661 }
 662
 663 static inline void free_pgtable_page(void *vaddr)
 664 {
 665         free_page((unsigned long)vaddr);
 666 }
 667
 668 static inline void *alloc_domain_mem(void)
 669 {
 670         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 671 }
 672
 673 static void free_domain_mem(void *vaddr)
 674 {
 675         kmem_cache_free(iommu_domain_cache, vaddr);
 676 }
 677
 678 static inline void * alloc_devinfo_mem(void)
 679 {
 680         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 681 }
 682
 683 static inline void free_devinfo_mem(void *vaddr)
 684 {
 685         kmem_cache_free(iommu_devinfo_cache, vaddr);
 686 }
 687
 688 static inline int domain_type_is_vm(struct dmar_domain *domain)
 689 {
 690         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
 691 }
 692
 693 static inline int domain_type_is_si(struct dmar_domain *domain)
 694 {
 695         return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
 696 }
 697
 698 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
 699 {
 700         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
 701                                 DOMAIN_FLAG_STATIC_IDENTITY);
 702 }
 703
 704 static inline int domain_pfn_supported(struct dmar_domain *domain,
 705                                        unsigned long pfn)
 706 {
 707         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 708
 709         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 710 }
 711
 712 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 713 {
 714         unsigned long sagaw;
 715         int agaw = -1;
 716
 717         sagaw = cap_sagaw(iommu->cap);
 718         for (agaw = width_to_agaw(max_gaw);
 719              agaw >= 0; agaw--) {
 720                 if (test_bit(agaw, &sagaw))
 721                         break;
 722         }
 723
 724         return agaw;
 725 }
 726
 727 /*
 728  * Calculate max SAGAW for each iommu.
 729  */
 730 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 731 {
 732         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 733 }
 734
 735 /*
 736  * calculate agaw for each iommu.
 737  * "SAGAW" may be different across iommus, use a default agaw, and
 738  * get a supported less agaw for iommus that don't support the default agaw.
 739  */
 740 int iommu_calculate_agaw(struct intel_iommu *iommu)
 741 {
 742         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 743 }
 744
 745 /* This functionin only returns single iommu in a domain */
 746 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 747 {
 748         int iommu_id;
 749
 750         /* si_domain and vm domain should not get here. */
 751         BUG_ON(domain_type_is_vm_or_si(domain));
 752         for_each_domain_iommu(iommu_id, domain)
 753                 break;
 754
 755         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 756                 return NULL;
 757
 758         return g_iommus[iommu_id];
 759 }
 760
 761 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 762 {
 763         struct dmar_drhd_unit *drhd;
 764         struct intel_iommu *iommu;
 765         bool found = false;
 766         int i;
 767
 768         domain->iommu_coherency = 1;
 769
 770         for_each_domain_iommu(i, domain) {
 771                 found = true;
 772                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 773                         domain->iommu_coherency = 0;
 774                         break;
 775                 }
 776         }
 777         if (found)
 778                 return;
 779
 780         /* No hardware attached; use lowest common denominator */
 781         rcu_read_lock();
 782         for_each_active_iommu(iommu, drhd) {
 783                 if (!ecap_coherent(iommu->ecap)) {
 784                         domain->iommu_coherency = 0;
 785                         break;
 786                 }
 787         }
 788         rcu_read_unlock();
 789 }
 790
 791 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 792 {
 793         struct dmar_drhd_unit *drhd;
 794         struct intel_iommu *iommu;
 795         int ret = 1;
 796
 797         rcu_read_lock();
 798         for_each_active_iommu(iommu, drhd) {
 799                 if (iommu != skip) {
 800                         if (!ecap_sc_support(iommu->ecap)) {
 801                                 ret = 0;
 802                                 break;
 803                         }
 804                 }
 805         }
 806         rcu_read_unlock();
 807
 808         return ret;
 809 }
 810
 811 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 812 {
 813         struct dmar_drhd_unit *drhd;
 814         struct intel_iommu *iommu;
 815         int mask = 0xf;
 816
 817         if (!intel_iommu_superpage) {
 818                 return 0;
 819         }
 820
 821         /* set iommu_superpage to the smallest common denominator */
 822         rcu_read_lock();
 823         for_each_active_iommu(iommu, drhd) {
 824                 if (iommu != skip) {
 825                         mask &= cap_super_page_val(iommu->cap);
 826                         if (!mask)
 827                                 break;
 828                 }
 829         }
 830         rcu_read_unlock();
 831
 832         return fls(mask);
 833 }
 834
 835 /* Some capabilities may be different across iommus */
 836 static void domain_update_iommu_cap(struct dmar_domain *domain)
 837 {
 838         domain_update_iommu_coherency(domain);
 839         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 840         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 841 }
 842
 843 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
 844                                                        u8 bus, u8 devfn, int alloc)
 845 {
 846         struct root_entry *root = &iommu->root_entry[bus];
 847         struct context_entry *context;
 848         u64 *entry;
 849
 850         entry = &root->lo;
 851         if (ecs_enabled(iommu)) {
 852                 if (devfn >= 0x80) {
 853                         devfn -= 0x80;
 854                         entry = &root->hi;
 855                 }
 856                 devfn *= 2;
 857         }
 858         if (*entry & 1)
 859                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 860         else {
 861                 unsigned long phy_addr;
 862                 if (!alloc)
 863                         return NULL;
 864
 865                 context = alloc_pgtable_page(iommu->node);
 866                 if (!context)
 867                         return NULL;
 868
 869                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 870                 phy_addr = virt_to_phys((void *)context);
 871                 *entry = phy_addr | 1;
 872                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 873         }
 874         return &context[devfn];
 875 }
 876
 877 static int iommu_dummy(struct device *dev)
 878 {
 879         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 880 }
 881
 882 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 883 {
 884         struct dmar_drhd_unit *drhd = NULL;
 885         struct intel_iommu *iommu;
 886         struct device *tmp;
 887         struct pci_dev *ptmp, *pdev = NULL;
 888         u16 segment = 0;
 889         int i;
 890
 891         if (iommu_dummy(dev))
 892                 return NULL;
 893
 894         if (dev_is_pci(dev)) {
 895                 struct pci_dev *pf_pdev;
 896
 897                 pdev = to_pci_dev(dev);
 898                 /* VFs aren't listed in scope tables; we need to look up
 899                  * the PF instead to find the IOMMU. */
 900                 pf_pdev = pci_physfn(pdev);
 901                 dev = &pf_pdev->dev;
 902                 segment = pci_domain_nr(pdev->bus);
 903         } else if (has_acpi_companion(dev))
 904                 dev = &ACPI_COMPANION(dev)->dev;
 905
 906         rcu_read_lock();
 907         for_each_active_iommu(iommu, drhd) {
 908                 if (pdev && segment != drhd->segment)
 909                         continue;
 910
 911                 for_each_active_dev_scope(drhd->devices,
 912                                           drhd->devices_cnt, i, tmp) {
 913                         if (tmp == dev) {
 914                                 /* For a VF use its original BDF# not that of the PF
 915                                  * which we used for the IOMMU lookup. Strictly speaking
 916                                  * we could do this for all PCI devices; we only need to
 917                                  * get the BDF# from the scope table for ACPI matches. */
 918                                 if (pdev->is_virtfn)
 919                                         goto got_pdev;
 920
 921                                 *bus = drhd->devices[i].bus;
 922                                 *devfn = drhd->devices[i].devfn;
 923                                 goto out;
 924                         }
 925
 926                         if (!pdev || !dev_is_pci(tmp))
 927                                 continue;
 928
 929                         ptmp = to_pci_dev(tmp);
 930                         if (ptmp->subordinate &&
 931                             ptmp->subordinate->number <= pdev->bus->number &&
 932                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 933                                 goto got_pdev;
 934                 }
 935
 936                 if (pdev && drhd->include_all) {
 937                 got_pdev:
 938                         *bus = pdev->bus->number;
 939                         *devfn = pdev->devfn;
 940                         goto out;
 941                 }
 942         }
 943         iommu = NULL;
 944  out:
 945         rcu_read_unlock();
 946
 947         return iommu;
 948 }
 949
 950 static void domain_flush_cache(struct dmar_domain *domain,
 951                                void *addr, int size)
 952 {
 953         if (!domain->iommu_coherency)
 954                 clflush_cache_range(addr, size);
 955 }
 956
 957 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 958 {
 959         struct context_entry *context;
 960         int ret = 0;
 961         unsigned long flags;
 962
 963         spin_lock_irqsave(&iommu->lock, flags);
 964         context = iommu_context_addr(iommu, bus, devfn, 0);
 965         if (context)
 966                 ret = context_present(context);
 967         spin_unlock_irqrestore(&iommu->lock, flags);
 968         return ret;
 969 }
 970
 971 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 972 {
 973         struct context_entry *context;
 974         unsigned long flags;
 975
 976         spin_lock_irqsave(&iommu->lock, flags);
 977         context = iommu_context_addr(iommu, bus, devfn, 0);
 978         if (context) {
 979                 context_clear_entry(context);
 980                 __iommu_flush_cache(iommu, context, sizeof(*context));
 981         }
 982         spin_unlock_irqrestore(&iommu->lock, flags);
 983 }
 984
 985 static void free_context_table(struct intel_iommu *iommu)
 986 {
 987         int i;
 988         unsigned long flags;
 989         struct context_entry *context;
 990
 991         spin_lock_irqsave(&iommu->lock, flags);
 992         if (!iommu->root_entry) {
 993                 goto out;
 994         }
 995         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 996                 context = iommu_context_addr(iommu, i, 0, 0);
 997                 if (context)
 998                         free_pgtable_page(context);
 999
1000                 if (!ecs_enabled(iommu))
1001                         continue;
1002
1003                 context = iommu_context_addr(iommu, i, 0x80, 0);
1004                 if (context)
1005                         free_pgtable_page(context);
1006
1007         }
1008         free_pgtable_page(iommu->root_entry);
1009         iommu->root_entry = NULL;
1010 out:
1011         spin_unlock_irqrestore(&iommu->lock, flags);
1012 }
1013
1014 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
1015                                       unsigned long pfn, int *target_level)
1016 {
1017         struct dma_pte *parent, *pte = NULL;
1018         int level = agaw_to_level(domain->agaw);
1019         int offset;
1020
1021         BUG_ON(!domain->pgd);
1022
1023         if (!domain_pfn_supported(domain, pfn))
1024                 /* Address beyond IOMMU's addressing capabilities. */
1025                 return NULL;
1026
1027         parent = domain->pgd;
1028
1029         while (1) {
1030                 void *tmp_page;
1031
1032                 offset = pfn_level_offset(pfn, level);
1033                 pte = &parent[offset];
1034                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1035                         break;
1036                 if (level == *target_level)
1037                         break;
1038
1039                 if (!dma_pte_present(pte)) {
1040                         uint64_t pteval;
1041
1042                         tmp_page = alloc_pgtable_page(domain->nid);
1043
1044                         if (!tmp_page)
1045                                 return NULL;
1046
1047                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1048                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1049                         if (cmpxchg64(&pte->val, 0ULL, pteval))
1050                                 /* Someone else set it while we were thinking; use theirs. */
1051                                 free_pgtable_page(tmp_page);
1052                         else
1053                                 domain_flush_cache(domain, pte, sizeof(*pte));
1054                 }
1055                 if (level == 1)
1056                         break;
1057
1058                 parent = phys_to_virt(dma_pte_addr(pte));
1059                 level--;
1060         }
1061
1062         if (!*target_level)
1063                 *target_level = level;
1064
1065         return pte;
1066 }
1067
1068
1069 /* return address's pte at specific level */
1070 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1071                                          unsigned long pfn,
1072                                          int level, int *large_page)
1073 {
1074         struct dma_pte *parent, *pte = NULL;
1075         int total = agaw_to_level(domain->agaw);
1076         int offset;
1077
1078         parent = domain->pgd;
1079         while (level <= total) {
1080                 offset = pfn_level_offset(pfn, total);
1081                 pte = &parent[offset];
1082                 if (level == total)
1083                         return pte;
1084
1085                 if (!dma_pte_present(pte)) {
1086                         *large_page = total;
1087                         break;
1088                 }
1089
1090                 if (dma_pte_superpage(pte)) {
1091                         *large_page = total;
1092                         return pte;
1093                 }
1094
1095                 parent = phys_to_virt(dma_pte_addr(pte));
1096                 total--;
1097         }
1098         return NULL;
1099 }
1100
1101 /* clear last level pte, a tlb flush should be followed */
1102 static void dma_pte_clear_range(struct dmar_domain *domain,
1103                                 unsigned long start_pfn,
1104                                 unsigned long last_pfn)
1105 {
1106         unsigned int large_page = 1;
1107         struct dma_pte *first_pte, *pte;
1108
1109         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1110         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1111         BUG_ON(start_pfn > last_pfn);
1112
1113         /* we don't need lock here; nobody else touches the iova range */
1114         do {
1115                 large_page = 1;
1116                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1117                 if (!pte) {
1118                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1119                         continue;
1120                 }
1121                 do {
1122                         dma_clear_pte(pte);
1123                         start_pfn += lvl_to_nr_pages(large_page);
1124                         pte++;
1125                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1126
1127                 domain_flush_cache(domain, first_pte,
1128                                    (void *)pte - (void *)first_pte);
1129
1130         } while (start_pfn && start_pfn <= last_pfn);
1131 }
1132
1133 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1134                                struct dma_pte *pte, unsigned long pfn,
1135                                unsigned long start_pfn, unsigned long last_pfn)
1136 {
1137         pfn = max(start_pfn, pfn);
1138         pte = &pte[pfn_level_offset(pfn, level)];
1139
1140         do {
1141                 unsigned long level_pfn;
1142                 struct dma_pte *level_pte;
1143
1144                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1145                         goto next;
1146
1147                 level_pfn = pfn & level_mask(level - 1);
1148                 level_pte = phys_to_virt(dma_pte_addr(pte));
1149
1150                 if (level > 2)
1151                         dma_pte_free_level(domain, level - 1, level_pte,
1152                                            level_pfn, start_pfn, last_pfn);
1153
1154                 /* If range covers entire pagetable, free it */
1155                 if (!(start_pfn > level_pfn ||
1156                       last_pfn < level_pfn + level_size(level) - 1)) {
1157                         dma_clear_pte(pte);
1158                         domain_flush_cache(domain, pte, sizeof(*pte));
1159                         free_pgtable_page(level_pte);
1160                 }
1161 next:
1162                 pfn += level_size(level);
1163         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1164 }
1165
1166 /* clear last level (leaf) ptes and free page table pages. */
1167 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1168                                    unsigned long start_pfn,
1169                                    unsigned long last_pfn)
1170 {
1171         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1172         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1173         BUG_ON(start_pfn > last_pfn);
1174
1175         dma_pte_clear_range(domain, start_pfn, last_pfn);
1176
1177         /* We don't need lock here; nobody else touches the iova range */
1178         dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1179                            domain->pgd, 0, start_pfn, last_pfn);
1180
1181         /* free pgd */
1182         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1183                 free_pgtable_page(domain->pgd);
1184                 domain->pgd = NULL;
1185         }
1186 }
1187
1188 /* When a page at a given level is being unlinked from its parent, we don't
1189    need to *modify* it at all. All we need to do is make a list of all the
1190    pages which can be freed just as soon as we've flushed the IOTLB and we
1191    know the hardware page-walk will no longer touch them.
1192    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1193    be freed. */
1194 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1195                                             int level, struct dma_pte *pte,
1196                                             struct page *freelist)
1197 {
1198         struct page *pg;
1199
1200         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1201         pg->freelist = freelist;
1202         freelist = pg;
1203
1204         if (level == 1)
1205                 return freelist;
1206
1207         pte = page_address(pg);
1208         do {
1209                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1210                         freelist = dma_pte_list_pagetables(domain, level - 1,
1211                                                            pte, freelist);
1212                 pte++;
1213         } while (!first_pte_in_page(pte));
1214
1215         return freelist;
1216 }
1217
1218 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1219                                         struct dma_pte *pte, unsigned long pfn,
1220                                         unsigned long start_pfn,
1221                                         unsigned long last_pfn,
1222                                         struct page *freelist)
1223 {
1224         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1225
1226         pfn = max(start_pfn, pfn);
1227         pte = &pte[pfn_level_offset(pfn, level)];
1228
1229         do {
1230                 unsigned long level_pfn;
1231
1232                 if (!dma_pte_present(pte))
1233                         goto next;
1234
1235                 level_pfn = pfn & level_mask(level);
1236
1237                 /* If range covers entire pagetable, free it */
1238                 if (start_pfn <= level_pfn &&
1239                     last_pfn >= level_pfn + level_size(level) - 1) {
1240                         /* These suborbinate page tables are going away entirely. Don't
1241                            bother to clear them; we're just going to *free* them. */
1242                         if (level > 1 && !dma_pte_superpage(pte))
1243                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1244
1245                         dma_clear_pte(pte);
1246                         if (!first_pte)
1247                                 first_pte = pte;
1248                         last_pte = pte;
1249                 } else if (level > 1) {
1250                         /* Recurse down into a level that isn't *entirely* obsolete */
1251                         freelist = dma_pte_clear_level(domain, level - 1,
1252                                                        phys_to_virt(dma_pte_addr(pte)),
1253                                                        level_pfn, start_pfn, last_pfn,
1254                                                        freelist);
1255                 }
1256 next:
1257                 pfn += level_size(level);
1258         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1259
1260         if (first_pte)
1261                 domain_flush_cache(domain, first_pte,
1262                                    (void *)++last_pte - (void *)first_pte);
1263
1264         return freelist;
1265 }
1266
1267 /* We can't just free the pages because the IOMMU may still be walking
1268    the page tables, and may have cached the intermediate levels. The
1269    pages can only be freed after the IOTLB flush has been done. */
1270 static struct page *domain_unmap(struct dmar_domain *domain,
1271                                  unsigned long start_pfn,
1272                                  unsigned long last_pfn)
1273 {
1274         struct page *freelist = NULL;
1275
1276         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1277         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1278         BUG_ON(start_pfn > last_pfn);
1279
1280         /* we don't need lock here; nobody else touches the iova range */
1281         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1282                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1283
1284         /* free pgd */
1285         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1286                 struct page *pgd_page = virt_to_page(domain->pgd);
1287                 pgd_page->freelist = freelist;
1288                 freelist = pgd_page;
1289
1290                 domain->pgd = NULL;
1291         }
1292
1293         return freelist;
1294 }
1295
1296 static void dma_free_pagelist(struct page *freelist)
1297 {
1298         struct page *pg;
1299
1300         while ((pg = freelist)) {
1301                 freelist = pg->freelist;
1302                 free_pgtable_page(page_address(pg));
1303         }
1304 }
1305
1306 /* iommu handling */
1307 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1308 {
1309         struct root_entry *root;
1310         unsigned long flags;
1311
1312         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1313         if (!root) {
1314                 pr_err("Allocating root entry for %s failed\n",
1315                         iommu->name);
1316                 return -ENOMEM;
1317         }
1318
1319         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1320
1321         spin_lock_irqsave(&iommu->lock, flags);
1322         iommu->root_entry = root;
1323         spin_unlock_irqrestore(&iommu->lock, flags);
1324
1325         return 0;
1326 }
1327
1328 static void iommu_set_root_entry(struct intel_iommu *iommu)
1329 {
1330         u64 addr;
1331         u32 sts;
1332         unsigned long flag;
1333
1334         addr = virt_to_phys(iommu->root_entry);
1335         if (ecs_enabled(iommu))
1336                 addr |= DMA_RTADDR_RTT;
1337
1338         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1339         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1340
1341         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1342
1343         /* Make sure hardware complete it */
1344         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1345                       readl, (sts & DMA_GSTS_RTPS), sts);
1346
1347         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1348 }
1349
1350 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1351 {
1352         u32 val;
1353         unsigned long flag;
1354
1355         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1356                 return;
1357
1358         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1359         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1360
1361         /* Make sure hardware complete it */
1362         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1363                       readl, (!(val & DMA_GSTS_WBFS)), val);
1364
1365         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1366 }
1367
1368 /* return value determine if we need a write buffer flush */
1369 static void __iommu_flush_context(struct intel_iommu *iommu,
1370                                   u16 did, u16 source_id, u8 function_mask,
1371                                   u64 type)
1372 {
1373         u64 val = 0;
1374         unsigned long flag;
1375
1376         switch (type) {
1377         case DMA_CCMD_GLOBAL_INVL:
1378                 val = DMA_CCMD_GLOBAL_INVL;
1379                 break;
1380         case DMA_CCMD_DOMAIN_INVL:
1381                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1382                 break;
1383         case DMA_CCMD_DEVICE_INVL:
1384                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1385                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1386                 break;
1387         default:
1388                 BUG();
1389         }
1390         val |= DMA_CCMD_ICC;
1391
1392         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1393         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1394
1395         /* Make sure hardware complete it */
1396         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1397                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1398
1399         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1400 }
1401
1402 /* return value determine if we need a write buffer flush */
1403 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1404                                 u64 addr, unsigned int size_order, u64 type)
1405 {
1406         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1407         u64 val = 0, val_iva = 0;
1408         unsigned long flag;
1409
1410         switch (type) {
1411         case DMA_TLB_GLOBAL_FLUSH:
1412                 /* global flush doesn't need set IVA_REG */
1413                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1414                 break;
1415         case DMA_TLB_DSI_FLUSH:
1416                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1417                 break;
1418         case DMA_TLB_PSI_FLUSH:
1419                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1420                 /* IH bit is passed in as part of address */
1421                 val_iva = size_order | addr;
1422                 break;
1423         default:
1424                 BUG();
1425         }
1426         /* Note: set drain read/write */
1427 #if 0
1428         /*
1429          * This is probably to be super secure.. Looks like we can
1430          * ignore it without any impact.
1431          */
1432         if (cap_read_drain(iommu->cap))
1433                 val |= DMA_TLB_READ_DRAIN;
1434 #endif
1435         if (cap_write_drain(iommu->cap))
1436                 val |= DMA_TLB_WRITE_DRAIN;
1437
1438         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1439         /* Note: Only uses first TLB reg currently */
1440         if (val_iva)
1441                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1442         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1443
1444         /* Make sure hardware complete it */
1445         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1446                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1447
1448         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1449
1450         /* check IOTLB invalidation granularity */
1451         if (DMA_TLB_IAIG(val) == 0)
1452                 pr_err("Flush IOTLB failed\n");
1453         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1454                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1455                         (unsigned long long)DMA_TLB_IIRG(type),
1456                         (unsigned long long)DMA_TLB_IAIG(val));
1457 }
1458
1459 static struct device_domain_info *
1460 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1461                          u8 bus, u8 devfn)
1462 {
1463         struct device_domain_info *info;
1464
1465         assert_spin_locked(&device_domain_lock);
1466
1467         if (!iommu->qi)
1468                 return NULL;
1469
1470         list_for_each_entry(info, &domain->devices, link)
1471                 if (info->iommu == iommu && info->bus == bus &&
1472                     info->devfn == devfn) {
1473                         if (info->ats_supported && info->dev)
1474                                 return info;
1475                         break;
1476                 }
1477
1478         return NULL;
1479 }
1480
1481 static void domain_update_iotlb(struct dmar_domain *domain)
1482 {
1483         struct device_domain_info *info;
1484         bool has_iotlb_device = false;
1485
1486         assert_spin_locked(&device_domain_lock);
1487
1488         list_for_each_entry(info, &domain->devices, link) {
1489                 struct pci_dev *pdev;
1490
1491                 if (!info->dev || !dev_is_pci(info->dev))
1492                         continue;
1493
1494                 pdev = to_pci_dev(info->dev);
1495                 if (pdev->ats_enabled) {
1496                         has_iotlb_device = true;
1497                         break;
1498                 }
1499         }
1500
1501         domain->has_iotlb_device = has_iotlb_device;
1502 }
1503
1504 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1505 {
1506         struct pci_dev *pdev;
1507
1508         assert_spin_locked(&device_domain_lock);
1509
1510         if (!info || !dev_is_pci(info->dev))
1511                 return;
1512
1513         pdev = to_pci_dev(info->dev);
1514
1515 #ifdef CONFIG_INTEL_IOMMU_SVM
1516         /* The PCIe spec, in its wisdom, declares that the behaviour of
1517            the device if you enable PASID support after ATS support is
1518            undefined. So always enable PASID support on devices which
1519            have it, even if we can't yet know if we're ever going to
1520            use it. */
1521         if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1522                 info->pasid_enabled = 1;
1523
1524         if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1525                 info->pri_enabled = 1;
1526 #endif
1527         if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1528                 info->ats_enabled = 1;
1529                 domain_update_iotlb(info->domain);
1530                 info->ats_qdep = pci_ats_queue_depth(pdev);
1531         }
1532 }
1533
1534 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1535 {
1536         struct pci_dev *pdev;
1537
1538         assert_spin_locked(&device_domain_lock);
1539
1540         if (!dev_is_pci(info->dev))
1541                 return;
1542
1543         pdev = to_pci_dev(info->dev);
1544
1545         if (info->ats_enabled) {
1546                 pci_disable_ats(pdev);
1547                 info->ats_enabled = 0;
1548                 domain_update_iotlb(info->domain);
1549         }
1550 #ifdef CONFIG_INTEL_IOMMU_SVM
1551         if (info->pri_enabled) {
1552                 pci_disable_pri(pdev);
1553                 info->pri_enabled = 0;
1554         }
1555         if (info->pasid_enabled) {
1556                 pci_disable_pasid(pdev);
1557                 info->pasid_enabled = 0;
1558         }
1559 #endif
1560 }
1561
1562 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1563                                   u64 addr, unsigned mask)
1564 {
1565         u16 sid, qdep;
1566         unsigned long flags;
1567         struct device_domain_info *info;
1568
1569         if (!domain->has_iotlb_device)
1570                 return;
1571
1572         spin_lock_irqsave(&device_domain_lock, flags);
1573         list_for_each_entry(info, &domain->devices, link) {
1574                 if (!info->ats_enabled)
1575                         continue;
1576
1577                 sid = info->bus << 8 | info->devfn;
1578                 qdep = info->ats_qdep;
1579                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1580         }
1581         spin_unlock_irqrestore(&device_domain_lock, flags);
1582 }
1583
1584 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1585                                   struct dmar_domain *domain,
1586                                   unsigned long pfn, unsigned int pages,
1587                                   int ih, int map)
1588 {
1589         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1590         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1591         u16 did = domain->iommu_did[iommu->seq_id];
1592
1593         BUG_ON(pages == 0);
1594
1595         if (ih)
1596                 ih = 1 << 6;
1597         /*
1598          * Fallback to domain selective flush if no PSI support or the size is
1599          * too big.
1600          * PSI requires page size to be 2 ^ x, and the base address is naturally
1601          * aligned to the size
1602          */
1603         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1604                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1605                                                 DMA_TLB_DSI_FLUSH);
1606         else
1607                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1608                                                 DMA_TLB_PSI_FLUSH);
1609
1610         /*
1611          * In caching mode, changes of pages from non-present to present require
1612          * flush. However, device IOTLB doesn't need to be flushed in this case.
1613          */
1614         if (!cap_caching_mode(iommu->cap) || !map)
1615                 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1616                                       addr, mask);
1617 }
1618
1619 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1620 {
1621         u32 pmen;
1622         unsigned long flags;
1623
1624         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1625         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1626         pmen &= ~DMA_PMEN_EPM;
1627         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1628
1629         /* wait for the protected region status bit to clear */
1630         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1631                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1632
1633         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1634 }
1635
1636 static void iommu_enable_translation(struct intel_iommu *iommu)
1637 {
1638         u32 sts;
1639         unsigned long flags;
1640
1641         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1642         iommu->gcmd |= DMA_GCMD_TE;
1643         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1644
1645         /* Make sure hardware complete it */
1646         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1647                       readl, (sts & DMA_GSTS_TES), sts);
1648
1649         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1650 }
1651
1652 static void iommu_disable_translation(struct intel_iommu *iommu)
1653 {
1654         u32 sts;
1655         unsigned long flag;
1656
1657         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1658         iommu->gcmd &= ~DMA_GCMD_TE;
1659         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1660
1661         /* Make sure hardware complete it */
1662         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1663                       readl, (!(sts & DMA_GSTS_TES)), sts);
1664
1665         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1666 }
1667
1668
1669 static int iommu_init_domains(struct intel_iommu *iommu)
1670 {
1671         u32 ndomains, nlongs;
1672         size_t size;
1673
1674         ndomains = cap_ndoms(iommu->cap);
1675         pr_debug("%s: Number of Domains supported <%d>\n",
1676                  iommu->name, ndomains);
1677         nlongs = BITS_TO_LONGS(ndomains);
1678
1679         spin_lock_init(&iommu->lock);
1680
1681         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1682         if (!iommu->domain_ids) {
1683                 pr_err("%s: Allocating domain id array failed\n",
1684                        iommu->name);
1685                 return -ENOMEM;
1686         }
1687
1688         size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1689         iommu->domains = kzalloc(size, GFP_KERNEL);
1690
1691         if (iommu->domains) {
1692                 size = 256 * sizeof(struct dmar_domain *);
1693                 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1694         }
1695
1696         if (!iommu->domains || !iommu->domains[0]) {
1697                 pr_err("%s: Allocating domain array failed\n",
1698                        iommu->name);
1699                 kfree(iommu->domain_ids);
1700                 kfree(iommu->domains);
1701                 iommu->domain_ids = NULL;
1702                 iommu->domains    = NULL;
1703                 return -ENOMEM;
1704         }
1705
1706
1707
1708         /*
1709          * If Caching mode is set, then invalid translations are tagged
1710          * with domain-id 0, hence we need to pre-allocate it. We also
1711          * use domain-id 0 as a marker for non-allocated domain-id, so
1712          * make sure it is not used for a real domain.
1713          */
1714         set_bit(0, iommu->domain_ids);
1715
1716         return 0;
1717 }
1718
1719 static void disable_dmar_iommu(struct intel_iommu *iommu)
1720 {
1721         struct device_domain_info *info, *tmp;
1722         unsigned long flags;
1723
1724         if (!iommu->domains || !iommu->domain_ids)
1725                 return;
1726
1727 again:
1728         spin_lock_irqsave(&device_domain_lock, flags);
1729         list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1730                 struct dmar_domain *domain;
1731
1732                 if (info->iommu != iommu)
1733                         continue;
1734
1735                 if (!info->dev || !info->domain)
1736                         continue;
1737
1738                 domain = info->domain;
1739
1740                 __dmar_remove_one_dev_info(info);
1741
1742                 if (!domain_type_is_vm_or_si(domain)) {
1743                         /*
1744                          * The domain_exit() function  can't be called under
1745                          * device_domain_lock, as it takes this lock itself.
1746                          * So release the lock here and re-run the loop
1747                          * afterwards.
1748                          */
1749                         spin_unlock_irqrestore(&device_domain_lock, flags);
1750                         domain_exit(domain);
1751                         goto again;
1752                 }
1753         }
1754         spin_unlock_irqrestore(&device_domain_lock, flags);
1755
1756         if (iommu->gcmd & DMA_GCMD_TE)
1757                 iommu_disable_translation(iommu);
1758 }
1759
1760 static void free_dmar_iommu(struct intel_iommu *iommu)
1761 {
1762         if ((iommu->domains) && (iommu->domain_ids)) {
1763                 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1764                 int i;
1765
1766                 for (i = 0; i < elems; i++)
1767                         kfree(iommu->domains[i]);
1768                 kfree(iommu->domains);
1769                 kfree(iommu->domain_ids);
1770                 iommu->domains = NULL;
1771                 iommu->domain_ids = NULL;
1772         }
1773
1774         g_iommus[iommu->seq_id] = NULL;
1775
1776         /* free context mapping */
1777         free_context_table(iommu);
1778
1779 #ifdef CONFIG_INTEL_IOMMU_SVM
1780         if (pasid_enabled(iommu)) {
1781                 if (ecap_prs(iommu->ecap))
1782                         intel_svm_finish_prq(iommu);
1783                 intel_svm_free_pasid_tables(iommu);
1784         }
1785 #endif
1786 }
1787
1788 static struct dmar_domain *alloc_domain(int flags)
1789 {
1790         struct dmar_domain *domain;
1791
1792         domain = alloc_domain_mem();
1793         if (!domain)
1794                 return NULL;
1795
1796         memset(domain, 0, sizeof(*domain));
1797         domain->nid = -1;
1798         domain->flags = flags;
1799         domain->has_iotlb_device = false;
1800         INIT_LIST_HEAD(&domain->devices);
1801
1802         return domain;
1803 }
1804
1805 /* Must be called with iommu->lock */
1806 static int domain_attach_iommu(struct dmar_domain *domain,
1807                                struct intel_iommu *iommu)
1808 {
1809         unsigned long ndomains;
1810         int num;
1811
1812         assert_spin_locked(&device_domain_lock);
1813         assert_spin_locked(&iommu->lock);
1814
1815         domain->iommu_refcnt[iommu->seq_id] += 1;
1816         domain->iommu_count += 1;
1817         if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1818                 ndomains = cap_ndoms(iommu->cap);
1819                 num      = find_first_zero_bit(iommu->domain_ids, ndomains);
1820
1821                 if (num >= ndomains) {
1822                         pr_err("%s: No free domain ids\n", iommu->name);
1823                         domain->iommu_refcnt[iommu->seq_id] -= 1;
1824                         domain->iommu_count -= 1;
1825                         return -ENOSPC;
1826                 }
1827
1828                 set_bit(num, iommu->domain_ids);
1829                 set_iommu_domain(iommu, num, domain);
1830
1831                 domain->iommu_did[iommu->seq_id] = num;
1832                 domain->nid                      = iommu->node;
1833
1834                 domain_update_iommu_cap(domain);
1835         }
1836
1837         return 0;
1838 }
1839
1840 static int domain_detach_iommu(struct dmar_domain *domain,
1841                                struct intel_iommu *iommu)
1842 {
1843         int num, count = INT_MAX;
1844
1845         assert_spin_locked(&device_domain_lock);
1846         assert_spin_locked(&iommu->lock);
1847
1848         domain->iommu_refcnt[iommu->seq_id] -= 1;
1849         count = --domain->iommu_count;
1850         if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1851                 num = domain->iommu_did[iommu->seq_id];
1852                 clear_bit(num, iommu->domain_ids);
1853                 set_iommu_domain(iommu, num, NULL);
1854
1855                 domain_update_iommu_cap(domain);
1856                 domain->iommu_did[iommu->seq_id] = 0;
1857         }
1858
1859         return count;
1860 }
1861
1862 static struct iova_domain reserved_iova_list;
1863 static struct lock_class_key reserved_rbtree_key;
1864
1865 static int dmar_init_reserved_ranges(void)
1866 {
1867         struct pci_dev *pdev = NULL;
1868         struct iova *iova;
1869         int i;
1870
1871         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1872                         DMA_32BIT_PFN);
1873
1874         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1875                 &reserved_rbtree_key);
1876
1877         /* IOAPIC ranges shouldn't be accessed by DMA */
1878         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1879                 IOVA_PFN(IOAPIC_RANGE_END));
1880         if (!iova) {
1881                 pr_err("Reserve IOAPIC range failed\n");
1882                 return -ENODEV;
1883         }
1884
1885         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1886         for_each_pci_dev(pdev) {
1887                 struct resource *r;
1888
1889                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1890                         r = &pdev->resource[i];
1891                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1892                                 continue;
1893                         iova = reserve_iova(&reserved_iova_list,
1894                                             IOVA_PFN(r->start),
1895                                             IOVA_PFN(r->end));
1896                         if (!iova) {
1897                                 pr_err("Reserve iova failed\n");
1898                                 return -ENODEV;
1899                         }
1900                 }
1901         }
1902         return 0;
1903 }
1904
1905 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1906 {
1907         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1908 }
1909
1910 static inline int guestwidth_to_adjustwidth(int gaw)
1911 {
1912         int agaw;
1913         int r = (gaw - 12) % 9;
1914
1915         if (r == 0)
1916                 agaw = gaw;
1917         else
1918                 agaw = gaw + 9 - r;
1919         if (agaw > 64)
1920                 agaw = 64;
1921         return agaw;
1922 }
1923
1924 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1925                        int guest_width)
1926 {
1927         int adjust_width, agaw;
1928         unsigned long sagaw;
1929
1930         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1931                         DMA_32BIT_PFN);
1932         domain_reserve_special_ranges(domain);
1933
1934         /* calculate AGAW */
1935         if (guest_width > cap_mgaw(iommu->cap))
1936                 guest_width = cap_mgaw(iommu->cap);
1937         domain->gaw = guest_width;
1938         adjust_width = guestwidth_to_adjustwidth(guest_width);
1939         agaw = width_to_agaw(adjust_width);
1940         sagaw = cap_sagaw(iommu->cap);
1941         if (!test_bit(agaw, &sagaw)) {
1942                 /* hardware doesn't support it, choose a bigger one */
1943                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1944                 agaw = find_next_bit(&sagaw, 5, agaw);
1945                 if (agaw >= 5)
1946                         return -ENODEV;
1947         }
1948         domain->agaw = agaw;
1949
1950         if (ecap_coherent(iommu->ecap))
1951                 domain->iommu_coherency = 1;
1952         else
1953                 domain->iommu_coherency = 0;
1954
1955         if (ecap_sc_support(iommu->ecap))
1956                 domain->iommu_snooping = 1;
1957         else
1958                 domain->iommu_snooping = 0;
1959
1960         if (intel_iommu_superpage)
1961                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1962         else
1963                 domain->iommu_superpage = 0;
1964
1965         domain->nid = iommu->node;
1966
1967         /* always allocate the top pgd */
1968         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1969         if (!domain->pgd)
1970                 return -ENOMEM;
1971         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1972         return 0;
1973 }
1974
1975 static void domain_exit(struct dmar_domain *domain)
1976 {
1977         struct page *freelist = NULL;
1978
1979         /* Domain 0 is reserved, so dont process it */
1980         if (!domain)
1981                 return;
1982
1983         /* Flush any lazy unmaps that may reference this domain */
1984         if (!intel_iommu_strict) {
1985                 int cpu;
1986
1987                 for_each_possible_cpu(cpu)
1988                         flush_unmaps_timeout(cpu);
1989         }
1990
1991         /* Remove associated devices and clear attached or cached domains */
1992         rcu_read_lock();
1993         domain_remove_dev_info(domain);
1994         rcu_read_unlock();
1995
1996         /* destroy iovas */
1997         put_iova_domain(&domain->iovad);
1998
1999         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2000
2001         dma_free_pagelist(freelist);
2002
2003         free_domain_mem(domain);
2004 }
2005
2006 static int domain_context_mapping_one(struct dmar_domain *domain,
2007                                       struct intel_iommu *iommu,
2008                                       u8 bus, u8 devfn)
2009 {
2010         u16 did = domain->iommu_did[iommu->seq_id];
2011         int translation = CONTEXT_TT_MULTI_LEVEL;
2012         struct device_domain_info *info = NULL;
2013         struct context_entry *context;
2014         unsigned long flags;
2015         struct dma_pte *pgd;
2016         int ret, agaw;
2017
2018         WARN_ON(did == 0);
2019
2020         if (hw_pass_through && domain_type_is_si(domain))
2021                 translation = CONTEXT_TT_PASS_THROUGH;
2022
2023         pr_debug("Set context mapping for %02x:%02x.%d\n",
2024                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2025
2026         BUG_ON(!domain->pgd);
2027
2028         spin_lock_irqsave(&device_domain_lock, flags);
2029         spin_lock(&iommu->lock);
2030
2031         ret = -ENOMEM;
2032         context = iommu_context_addr(iommu, bus, devfn, 1);
2033         if (!context)
2034                 goto out_unlock;
2035
2036         ret = 0;
2037         if (context_present(context))
2038                 goto out_unlock;
2039
2040         /*
2041          * For kdump cases, old valid entries may be cached due to the
2042          * in-flight DMA and copied pgtable, but there is no unmapping
2043          * behaviour for them, thus we need an explicit cache flush for
2044          * the newly-mapped device. For kdump, at this point, the device
2045          * is supposed to finish reset at its driver probe stage, so no
2046          * in-flight DMA will exist, and we don't need to worry anymore
2047          * hereafter.
2048          */
2049         if (context_copied(context)) {
2050                 u16 did_old = context_domain_id(context);
2051
2052                 if (did_old >= 0 && did_old < cap_ndoms(iommu->cap))
2053                         iommu->flush.flush_context(iommu, did_old,
2054                                                    (((u16)bus) << 8) | devfn,
2055                                                    DMA_CCMD_MASK_NOBIT,
2056                                                    DMA_CCMD_DEVICE_INVL);
2057         }
2058
2059         pgd = domain->pgd;
2060
2061         context_clear_entry(context);
2062         context_set_domain_id(context, did);
2063
2064         /*
2065          * Skip top levels of page tables for iommu which has less agaw
2066          * than default.  Unnecessary for PT mode.
2067          */
2068         if (translation != CONTEXT_TT_PASS_THROUGH) {
2069                 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
2070                         ret = -ENOMEM;
2071                         pgd = phys_to_virt(dma_pte_addr(pgd));
2072                         if (!dma_pte_present(pgd))
2073                                 goto out_unlock;
2074                 }
2075
2076                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2077                 if (info && info->ats_supported)
2078                         translation = CONTEXT_TT_DEV_IOTLB;
2079                 else
2080                         translation = CONTEXT_TT_MULTI_LEVEL;
2081
2082                 context_set_address_root(context, virt_to_phys(pgd));
2083                 context_set_address_width(context, iommu->agaw);
2084         } else {
2085                 /*
2086                  * In pass through mode, AW must be programmed to
2087                  * indicate the largest AGAW value supported by
2088                  * hardware. And ASR is ignored by hardware.
2089                  */
2090                 context_set_address_width(context, iommu->msagaw);
2091         }
2092
2093         context_set_translation_type(context, translation);
2094         context_set_fault_enable(context);
2095         context_set_present(context);
2096         domain_flush_cache(domain, context, sizeof(*context));
2097
2098         /*
2099          * It's a non-present to present mapping. If hardware doesn't cache
2100          * non-present entry we only need to flush the write-buffer. If the
2101          * _does_ cache non-present entries, then it does so in the special
2102          * domain #0, which we have to flush:
2103          */
2104         if (cap_caching_mode(iommu->cap)) {
2105                 iommu->flush.flush_context(iommu, 0,
2106                                            (((u16)bus) << 8) | devfn,
2107                                            DMA_CCMD_MASK_NOBIT,
2108                                            DMA_CCMD_DEVICE_INVL);
2109                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2110         } else {
2111                 iommu_flush_write_buffer(iommu);
2112         }
2113         iommu_enable_dev_iotlb(info);
2114
2115         ret = 0;
2116
2117 out_unlock:
2118         spin_unlock(&iommu->lock);
2119         spin_unlock_irqrestore(&device_domain_lock, flags);
2120
2121         return ret;
2122 }
2123
2124 struct domain_context_mapping_data {
2125         struct dmar_domain *domain;
2126         struct intel_iommu *iommu;
2127 };
2128
2129 static int domain_context_mapping_cb(struct pci_dev *pdev,
2130                                      u16 alias, void *opaque)
2131 {
2132         struct domain_context_mapping_data *data = opaque;
2133
2134         return domain_context_mapping_one(data->domain, data->iommu,
2135                                           PCI_BUS_NUM(alias), alias & 0xff);
2136 }
2137
2138 static int
2139 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2140 {
2141         struct intel_iommu *iommu;
2142         u8 bus, devfn;
2143         struct domain_context_mapping_data data;
2144
2145         iommu = device_to_iommu(dev, &bus, &devfn);
2146         if (!iommu)
2147                 return -ENODEV;
2148
2149         if (!dev_is_pci(dev))
2150                 return domain_context_mapping_one(domain, iommu, bus, devfn);
2151
2152         data.domain = domain;
2153         data.iommu = iommu;
2154
2155         return pci_for_each_dma_alias(to_pci_dev(dev),
2156                                       &domain_context_mapping_cb, &data);
2157 }
2158
2159 static int domain_context_mapped_cb(struct pci_dev *pdev,
2160                                     u16 alias, void *opaque)
2161 {
2162         struct intel_iommu *iommu = opaque;
2163
2164         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2165 }
2166
2167 static int domain_context_mapped(struct device *dev)
2168 {
2169         struct intel_iommu *iommu;
2170         u8 bus, devfn;
2171
2172         iommu = device_to_iommu(dev, &bus, &devfn);
2173         if (!iommu)
2174                 return -ENODEV;
2175
2176         if (!dev_is_pci(dev))
2177                 return device_context_mapped(iommu, bus, devfn);
2178
2179         return !pci_for_each_dma_alias(to_pci_dev(dev),
2180                                        domain_context_mapped_cb, iommu);
2181 }
2182
2183 /* Returns a number of VTD pages, but aligned to MM page size */
2184 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2185                                             size_t size)
2186 {
2187         host_addr &= ~PAGE_MASK;
2188         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2189 }
2190
2191 /* Return largest possible superpage level for a given mapping */
2192 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2193                                           unsigned long iov_pfn,
2194                                           unsigned long phy_pfn,
2195                                           unsigned long pages)
2196 {
2197         int support, level = 1;
2198         unsigned long pfnmerge;
2199
2200         support = domain->iommu_superpage;
2201
2202         /* To use a large page, the virtual *and* physical addresses
2203            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2204            of them will mean we have to use smaller pages. So just
2205            merge them and check both at once. */
2206         pfnmerge = iov_pfn | phy_pfn;
2207
2208         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2209                 pages >>= VTD_STRIDE_SHIFT;
2210                 if (!pages)
2211                         break;
2212                 pfnmerge >>= VTD_STRIDE_SHIFT;
2213                 level++;
2214                 support--;
2215         }
2216         return level;
2217 }
2218
2219 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2220                             struct scatterlist *sg, unsigned long phys_pfn,
2221                             unsigned long nr_pages, int prot)
2222 {
2223         struct dma_pte *first_pte = NULL, *pte = NULL;
2224         phys_addr_t uninitialized_var(pteval);
2225         unsigned long sg_res = 0;
2226         unsigned int largepage_lvl = 0;
2227         unsigned long lvl_pages = 0;
2228
2229         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2230
2231         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2232                 return -EINVAL;
2233
2234         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2235
2236         if (!sg) {
2237                 sg_res = nr_pages;
2238                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2239         }
2240
2241         while (nr_pages > 0) {
2242                 uint64_t tmp;
2243
2244                 if (!sg_res) {
2245                         sg_res = aligned_nrpages(sg->offset, sg->length);
2246                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2247                         sg->dma_length = sg->length;
2248                         pteval = page_to_phys(sg_page(sg)) | prot;
2249                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2250                 }
2251
2252                 if (!pte) {
2253                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2254
2255                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2256                         if (!pte)
2257                                 return -ENOMEM;
2258                         /* It is large page*/
2259                         if (largepage_lvl > 1) {
2260                                 unsigned long nr_superpages, end_pfn;
2261
2262                                 pteval |= DMA_PTE_LARGE_PAGE;
2263                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2264
2265                                 nr_superpages = sg_res / lvl_pages;
2266                                 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2267
2268                                 /*
2269                                  * Ensure that old small page tables are
2270                                  * removed to make room for superpage(s).
2271                                  */
2272                                 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
2273                         } else {
2274                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2275                         }
2276
2277                 }
2278                 /* We don't need lock here, nobody else
2279                  * touches the iova range
2280                  */
2281                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2282                 if (tmp) {
2283                         static int dumps = 5;
2284                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2285                                 iov_pfn, tmp, (unsigned long long)pteval);
2286                         if (dumps) {
2287                                 dumps--;
2288                                 debug_dma_dump_mappings(NULL);
2289                         }
2290                         WARN_ON(1);
2291                 }
2292
2293                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2294
2295                 BUG_ON(nr_pages < lvl_pages);
2296                 BUG_ON(sg_res < lvl_pages);
2297
2298                 nr_pages -= lvl_pages;
2299                 iov_pfn += lvl_pages;
2300                 phys_pfn += lvl_pages;
2301                 pteval += lvl_pages * VTD_PAGE_SIZE;
2302                 sg_res -= lvl_pages;
2303
2304                 /* If the next PTE would be the first in a new page, then we
2305                    need to flush the cache on the entries we've just written.
2306                    And then we'll need to recalculate 'pte', so clear it and
2307                    let it get set again in the if (!pte) block above.
2308
2309                    If we're done (!nr_pages) we need to flush the cache too.
2310
2311                    Also if we've been setting superpages, we may need to
2312                    recalculate 'pte' and switch back to smaller pages for the
2313                    end of the mapping, if the trailing size is not enough to
2314                    use another superpage (i.e. sg_res < lvl_pages). */
2315                 pte++;
2316                 if (!nr_pages || first_pte_in_page(pte) ||
2317                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2318                         domain_flush_cache(domain, first_pte,
2319                                            (void *)pte - (void *)first_pte);
2320                         pte = NULL;
2321                 }
2322
2323                 if (!sg_res && nr_pages)
2324                         sg = sg_next(sg);
2325         }
2326         return 0;
2327 }
2328
2329 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2330                                     struct scatterlist *sg, unsigned long nr_pages,
2331                                     int prot)
2332 {
2333         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2334 }
2335
2336 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2337                                      unsigned long phys_pfn, unsigned long nr_pages,
2338                                      int prot)
2339 {
2340         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2341 }
2342
2343 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2344 {
2345         if (!iommu)
2346                 return;
2347
2348         clear_context_table(iommu, bus, devfn);
2349         iommu->flush.flush_context(iommu, 0, 0, 0,
2350                                            DMA_CCMD_GLOBAL_INVL);
2351         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2352 }
2353
2354 static inline void unlink_domain_info(struct device_domain_info *info)
2355 {
2356         assert_spin_locked(&device_domain_lock);
2357         list_del(&info->link);
2358         list_del(&info->global);
2359         if (info->dev)
2360                 info->dev->archdata.iommu = NULL;
2361 }
2362
2363 static void domain_remove_dev_info(struct dmar_domain *domain)
2364 {
2365         struct device_domain_info *info, *tmp;
2366         unsigned long flags;
2367
2368         spin_lock_irqsave(&device_domain_lock, flags);
2369         list_for_each_entry_safe(info, tmp, &domain->devices, link)
2370                 __dmar_remove_one_dev_info(info);
2371         spin_unlock_irqrestore(&device_domain_lock, flags);
2372 }
2373
2374 /*
2375  * find_domain
2376  * Note: we use struct device->archdata.iommu stores the info
2377  */
2378 static struct dmar_domain *find_domain(struct device *dev)
2379 {
2380         struct device_domain_info *info;
2381
2382         /* No lock here, assumes no domain exit in normal case */
2383         info = dev->archdata.iommu;
2384         if (info)
2385                 return info->domain;
2386         return NULL;
2387 }
2388
2389 static inline struct device_domain_info *
2390 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2391 {
2392         struct device_domain_info *info;
2393
2394         list_for_each_entry(info, &device_domain_list, global)
2395                 if (info->iommu->segment == segment && info->bus == bus &&
2396                     info->devfn == devfn)
2397                         return info;
2398
2399         return NULL;
2400 }
2401
2402 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2403                                                     int bus, int devfn,
2404                                                     struct device *dev,
2405                                                     struct dmar_domain *domain)
2406 {
2407         struct dmar_domain *found = NULL;
2408         struct device_domain_info *info;
2409         unsigned long flags;
2410         int ret;
2411
2412         info = alloc_devinfo_mem();
2413         if (!info)
2414                 return NULL;
2415
2416         info->bus = bus;
2417         info->devfn = devfn;
2418         info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2419         info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2420         info->ats_qdep = 0;
2421         info->dev = dev;
2422         info->domain = domain;
2423         info->iommu = iommu;
2424
2425         if (dev && dev_is_pci(dev)) {
2426                 struct pci_dev *pdev = to_pci_dev(info->dev);
2427
2428                 if (ecap_dev_iotlb_support(iommu->ecap) &&
2429                     pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2430                     dmar_find_matched_atsr_unit(pdev))
2431                         info->ats_supported = 1;
2432
2433                 if (ecs_enabled(iommu)) {
2434                         if (pasid_enabled(iommu)) {
2435                                 int features = pci_pasid_features(pdev);
2436                                 if (features >= 0)
2437                                         info->pasid_supported = features | 1;
2438                         }
2439
2440                         if (info->ats_supported && ecap_prs(iommu->ecap) &&
2441                             pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2442                                 info->pri_supported = 1;
2443                 }
2444         }
2445
2446         spin_lock_irqsave(&device_domain_lock, flags);
2447         if (dev)
2448                 found = find_domain(dev);
2449
2450         if (!found) {
2451                 struct device_domain_info *info2;
2452                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2453                 if (info2) {
2454                         found      = info2->domain;
2455                         info2->dev = dev;
2456                 }
2457         }
2458
2459         if (found) {
2460                 spin_unlock_irqrestore(&device_domain_lock, flags);
2461                 free_devinfo_mem(info);
2462                 /* Caller must free the original domain */
2463                 return found;
2464         }
2465
2466         spin_lock(&iommu->lock);
2467         ret = domain_attach_iommu(domain, iommu);
2468         spin_unlock(&iommu->lock);
2469
2470         if (ret) {
2471                 spin_unlock_irqrestore(&device_domain_lock, flags);
2472                 free_devinfo_mem(info);
2473                 return NULL;
2474         }
2475
2476         list_add(&info->link, &domain->devices);
2477         list_add(&info->global, &device_domain_list);
2478         if (dev)
2479                 dev->archdata.iommu = info;
2480         spin_unlock_irqrestore(&device_domain_lock, flags);
2481
2482         if (dev && domain_context_mapping(domain, dev)) {
2483                 pr_err("Domain context map for %s failed\n", dev_name(dev));
2484                 dmar_remove_one_dev_info(domain, dev);
2485                 return NULL;
2486         }
2487
2488         return domain;
2489 }
2490
2491 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2492 {
2493         *(u16 *)opaque = alias;
2494         return 0;
2495 }
2496
2497 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2498 {
2499         struct device_domain_info *info = NULL;
2500         struct dmar_domain *domain = NULL;
2501         struct intel_iommu *iommu;
2502         u16 req_id, dma_alias;
2503         unsigned long flags;
2504         u8 bus, devfn;
2505
2506         iommu = device_to_iommu(dev, &bus, &devfn);
2507         if (!iommu)
2508                 return NULL;
2509
2510         req_id = ((u16)bus << 8) | devfn;
2511
2512         if (dev_is_pci(dev)) {
2513                 struct pci_dev *pdev = to_pci_dev(dev);
2514
2515                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2516
2517                 spin_lock_irqsave(&device_domain_lock, flags);
2518                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2519                                                       PCI_BUS_NUM(dma_alias),
2520                                                       dma_alias & 0xff);
2521                 if (info) {
2522                         iommu = info->iommu;
2523                         domain = info->domain;
2524                 }
2525                 spin_unlock_irqrestore(&device_domain_lock, flags);
2526
2527                 /* DMA alias already has a domain, use it */
2528                 if (info)
2529                         goto out;
2530         }
2531
2532         /* Allocate and initialize new domain for the device */
2533         domain = alloc_domain(0);
2534         if (!domain)
2535                 return NULL;
2536         if (domain_init(domain, iommu, gaw)) {
2537                 domain_exit(domain);
2538                 return NULL;
2539         }
2540
2541 out:
2542
2543         return domain;
2544 }
2545
2546 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2547                                               struct dmar_domain *domain)
2548 {
2549         struct intel_iommu *iommu;
2550         struct dmar_domain *tmp;
2551         u16 req_id, dma_alias;
2552         u8 bus, devfn;
2553
2554         iommu = device_to_iommu(dev, &bus, &devfn);
2555         if (!iommu)
2556                 return NULL;
2557
2558         req_id = ((u16)bus << 8) | devfn;
2559
2560         if (dev_is_pci(dev)) {
2561                 struct pci_dev *pdev = to_pci_dev(dev);
2562
2563                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2564
2565                 /* register PCI DMA alias device */
2566                 if (req_id != dma_alias) {
2567                         tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2568                                         dma_alias & 0xff, NULL, domain);
2569
2570                         if (!tmp || tmp != domain)
2571                                 return tmp;
2572                 }
2573         }
2574
2575         tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2576         if (!tmp || tmp != domain)
2577                 return tmp;
2578
2579         return domain;
2580 }
2581
2582 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2583 {
2584         struct dmar_domain *domain, *tmp;
2585
2586         domain = find_domain(dev);
2587         if (domain)
2588                 goto out;
2589
2590         domain = find_or_alloc_domain(dev, gaw);
2591         if (!domain)
2592                 goto out;
2593
2594         tmp = set_domain_for_dev(dev, domain);
2595         if (!tmp || domain != tmp) {
2596                 domain_exit(domain);
2597                 domain = tmp;
2598         }
2599
2600 out:
2601
2602         return domain;
2603 }
2604
2605 static int iommu_domain_identity_map(struct dmar_domain *domain,
2606                                      unsigned long long start,
2607                                      unsigned long long end)
2608 {
2609         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2610         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2611
2612         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2613                           dma_to_mm_pfn(last_vpfn))) {
2614                 pr_err("Reserving iova failed\n");
2615                 return -ENOMEM;
2616         }
2617
2618         pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2619         /*
2620          * RMRR range might have overlap with physical memory range,
2621          * clear it first
2622          */
2623         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2624
2625         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2626                                   last_vpfn - first_vpfn + 1,
2627                                   DMA_PTE_READ|DMA_PTE_WRITE);
2628 }
2629
2630 static int domain_prepare_identity_map(struct device *dev,
2631                                        struct dmar_domain *domain,
2632                                        unsigned long long start,
2633                                        unsigned long long end)
2634 {
2635         /* For _hardware_ passthrough, don't bother. But for software
2636            passthrough, we do it anyway -- it may indicate a memory
2637            range which is reserved in E820, so which didn't get set
2638            up to start with in si_domain */
2639         if (domain == si_domain && hw_pass_through) {
2640                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2641                         dev_name(dev), start, end);
2642                 return 0;
2643         }
2644
2645         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2646                 dev_name(dev), start, end);
2647
2648         if (end < start) {
2649                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2650                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2651                         dmi_get_system_info(DMI_BIOS_VENDOR),
2652                         dmi_get_system_info(DMI_BIOS_VERSION),
2653                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2654                 return -EIO;
2655         }
2656
2657         if (end >> agaw_to_width(domain->agaw)) {
2658                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2659                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2660                      agaw_to_width(domain->agaw),
2661                      dmi_get_system_info(DMI_BIOS_VENDOR),
2662                      dmi_get_system_info(DMI_BIOS_VERSION),
2663                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2664                 return -EIO;
2665         }
2666
2667         return iommu_domain_identity_map(domain, start, end);
2668 }
2669
2670 static int iommu_prepare_identity_map(struct device *dev,
2671                                       unsigned long long start,
2672                                       unsigned long long end)
2673 {
2674         struct dmar_domain *domain;
2675         int ret;
2676
2677         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2678         if (!domain)
2679                 return -ENOMEM;
2680
2681         ret = domain_prepare_identity_map(dev, domain, start, end);
2682         if (ret)
2683                 domain_exit(domain);
2684
2685         return ret;
2686 }
2687
2688 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2689                                          struct device *dev)
2690 {
2691         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2692                 return 0;
2693         return iommu_prepare_identity_map(dev, rmrr->base_address,
2694                                           rmrr->end_address);
2695 }
2696
2697 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2698 static inline void iommu_prepare_isa(void)
2699 {
2700         struct pci_dev *pdev;
2701         int ret;
2702
2703         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2704         if (!pdev)
2705                 return;
2706
2707         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2708         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2709
2710         if (ret)
2711                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2712
2713         pci_dev_put(pdev);
2714 }
2715 #else
2716 static inline void iommu_prepare_isa(void)
2717 {
2718         return;
2719 }
2720 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2721
2722 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2723
2724 static int __init si_domain_init(int hw)
2725 {
2726         int nid, ret = 0;
2727
2728         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2729         if (!si_domain)
2730                 return -EFAULT;
2731
2732         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2733                 domain_exit(si_domain);
2734                 return -EFAULT;
2735         }
2736
2737         pr_debug("Identity mapping domain allocated\n");
2738
2739         if (hw)
2740                 return 0;
2741
2742         for_each_online_node(nid) {
2743                 unsigned long start_pfn, end_pfn;
2744                 int i;
2745
2746                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2747                         ret = iommu_domain_identity_map(si_domain,
2748                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2749                         if (ret)
2750                                 return ret;
2751                 }
2752         }
2753
2754         return 0;
2755 }
2756
2757 static int identity_mapping(struct device *dev)
2758 {
2759         struct device_domain_info *info;
2760
2761         if (likely(!iommu_identity_mapping))
2762                 return 0;
2763
2764         info = dev->archdata.iommu;
2765         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2766                 return (info->domain == si_domain);
2767
2768         return 0;
2769 }
2770
2771 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2772 {
2773         struct dmar_domain *ndomain;
2774         struct intel_iommu *iommu;
2775         u8 bus, devfn;
2776
2777         iommu = device_to_iommu(dev, &bus, &devfn);
2778         if (!iommu)
2779                 return -ENODEV;
2780
2781         ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2782         if (ndomain != domain)
2783                 return -EBUSY;
2784
2785         return 0;
2786 }
2787
2788 static bool device_has_rmrr(struct device *dev)
2789 {
2790         struct dmar_rmrr_unit *rmrr;
2791         struct device *tmp;
2792         int i;
2793
2794         rcu_read_lock();
2795         for_each_rmrr_units(rmrr) {
2796                 /*
2797                  * Return TRUE if this RMRR contains the device that
2798                  * is passed in.
2799                  */
2800                 for_each_active_dev_scope(rmrr->devices,
2801                                           rmrr->devices_cnt, i, tmp)
2802                         if (tmp == dev) {
2803                                 rcu_read_unlock();
2804                                 return true;
2805                         }
2806         }
2807         rcu_read_unlock();
2808         return false;
2809 }
2810
2811 /*
2812  * There are a couple cases where we need to restrict the functionality of
2813  * devices associated with RMRRs.  The first is when evaluating a device for
2814  * identity mapping because problems exist when devices are moved in and out
2815  * of domains and their respective RMRR information is lost.  This means that
2816  * a device with associated RMRRs will never be in a "passthrough" domain.
2817  * The second is use of the device through the IOMMU API.  This interface
2818  * expects to have full control of the IOVA space for the device.  We cannot
2819  * satisfy both the requirement that RMRR access is maintained and have an
2820  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2821  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2822  * We therefore prevent devices associated with an RMRR from participating in
2823  * the IOMMU API, which eliminates them from device assignment.
2824  *
2825  * In both cases we assume that PCI USB devices with RMRRs have them largely
2826  * for historical reasons and that the RMRR space is not actively used post
2827  * boot.  This exclusion may change if vendors begin to abuse it.
2828  *
2829  * The same exception is made for graphics devices, with the requirement that
2830  * any use of the RMRR regions will be torn down before assigning the device
2831  * to a guest.
2832  */
2833 static bool device_is_rmrr_locked(struct device *dev)
2834 {
2835         if (!device_has_rmrr(dev))
2836                 return false;
2837
2838         if (dev_is_pci(dev)) {
2839                 struct pci_dev *pdev = to_pci_dev(dev);
2840
2841                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2842                         return false;
2843         }
2844
2845         return true;
2846 }
2847
2848 static int iommu_should_identity_map(struct device *dev, int startup)
2849 {
2850
2851         if (dev_is_pci(dev)) {
2852                 struct pci_dev *pdev = to_pci_dev(dev);
2853
2854                 if (device_is_rmrr_locked(dev))
2855                         return 0;
2856
2857                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2858                         return 1;
2859
2860                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2861                         return 1;
2862
2863                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2864                         return 0;
2865
2866                 /*
2867                  * We want to start off with all devices in the 1:1 domain, and
2868                  * take them out later if we find they can't access all of memory.
2869                  *
2870                  * However, we can't do this for PCI devices behind bridges,
2871                  * because all PCI devices behind the same bridge will end up
2872                  * with the same source-id on their transactions.
2873                  *
2874                  * Practically speaking, we can't change things around for these
2875                  * devices at run-time, because we can't be sure there'll be no
2876                  * DMA transactions in flight for any of their siblings.
2877                  *
2878                  * So PCI devices (unless they're on the root bus) as well as
2879                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2880                  * the 1:1 domain, just in _case_ one of their siblings turns out
2881                  * not to be able to map all of memory.
2882                  */
2883                 if (!pci_is_pcie(pdev)) {
2884                         if (!pci_is_root_bus(pdev->bus))
2885                                 return 0;
2886                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2887                                 return 0;
2888                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2889                         return 0;
2890         } else {
2891                 if (device_has_rmrr(dev))
2892                         return 0;
2893         }
2894
2895         /*
2896          * At boot time, we don't yet know if devices will be 64-bit capable.
2897          * Assume that they will — if they turn out not to be, then we can
2898          * take them out of the 1:1 domain later.
2899          */
2900         if (!startup) {
2901                 /*
2902                  * If the device's dma_mask is less than the system's memory
2903                  * size then this is not a candidate for identity mapping.
2904                  */
2905                 u64 dma_mask = *dev->dma_mask;
2906
2907                 if (dev->coherent_dma_mask &&
2908                     dev->coherent_dma_mask < dma_mask)
2909                         dma_mask = dev->coherent_dma_mask;
2910
2911                 return dma_mask >= dma_get_required_mask(dev);
2912         }
2913
2914         return 1;
2915 }
2916
2917 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2918 {
2919         int ret;
2920
2921         if (!iommu_should_identity_map(dev, 1))
2922                 return 0;
2923
2924         ret = domain_add_dev_info(si_domain, dev);
2925         if (!ret)
2926                 pr_info("%s identity mapping for device %s\n",
2927                         hw ? "Hardware" : "Software", dev_name(dev));
2928         else if (ret == -ENODEV)
2929                 /* device not associated with an iommu */
2930                 ret = 0;
2931
2932         return ret;
2933 }
2934
2935
2936 static int __init iommu_prepare_static_identity_mapping(int hw)
2937 {
2938         struct pci_dev *pdev = NULL;
2939         struct dmar_drhd_unit *drhd;
2940         struct intel_iommu *iommu;
2941         struct device *dev;
2942         int i;
2943         int ret = 0;
2944
2945         for_each_pci_dev(pdev) {
2946                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2947                 if (ret)
2948                         return ret;
2949         }
2950
2951         for_each_active_iommu(iommu, drhd)
2952                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2953                         struct acpi_device_physical_node *pn;
2954                         struct acpi_device *adev;
2955
2956                         if (dev->bus != &acpi_bus_type)
2957                                 continue;
2958
2959                         adev= to_acpi_device(dev);
2960                         mutex_lock(&adev->physical_node_lock);
2961                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2962                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2963                                 if (ret)
2964                                         break;
2965                         }
2966                         mutex_unlock(&adev->physical_node_lock);
2967                         if (ret)
2968                                 return ret;
2969                 }
2970
2971         return 0;
2972 }
2973
2974 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2975 {
2976         /*
2977          * Start from the sane iommu hardware state.
2978          * If the queued invalidation is already initialized by us
2979          * (for example, while enabling interrupt-remapping) then
2980          * we got the things already rolling from a sane state.
2981          */
2982         if (!iommu->qi) {
2983                 /*
2984                  * Clear any previous faults.
2985                  */
2986                 dmar_fault(-1, iommu);
2987                 /*
2988                  * Disable queued invalidation if supported and already enabled
2989                  * before OS handover.
2990                  */
2991                 dmar_disable_qi(iommu);
2992         }
2993
2994         if (dmar_enable_qi(iommu)) {
2995                 /*
2996                  * Queued Invalidate not enabled, use Register Based Invalidate
2997                  */
2998                 iommu->flush.flush_context = __iommu_flush_context;
2999                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3000                 pr_info("%s: Using Register based invalidation\n",
3001                         iommu->name);
3002         } else {
3003                 iommu->flush.flush_context = qi_flush_context;
3004                 iommu->flush.flush_iotlb = qi_flush_iotlb;
3005                 pr_info("%s: Using Queued invalidation\n", iommu->name);
3006         }
3007 }
3008
3009 static int copy_context_table(struct intel_iommu *iommu,
3010                               struct root_entry *old_re,
3011                               struct context_entry **tbl,
3012                               int bus, bool ext)
3013 {
3014         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3015         struct context_entry *new_ce = NULL, ce;
3016         struct context_entry *old_ce = NULL;
3017         struct root_entry re;
3018         phys_addr_t old_ce_phys;
3019
3020         tbl_idx = ext ? bus * 2 : bus;
3021         memcpy(&re, old_re, sizeof(re));
3022
3023         for (devfn = 0; devfn < 256; devfn++) {
3024                 /* First calculate the correct index */
3025                 idx = (ext ? devfn * 2 : devfn) % 256;
3026
3027                 if (idx == 0) {
3028                         /* First save what we may have and clean up */
3029                         if (new_ce) {
3030                                 tbl[tbl_idx] = new_ce;
3031                                 __iommu_flush_cache(iommu, new_ce,
3032                                                     VTD_PAGE_SIZE);
3033                                 pos = 1;
3034                         }
3035
3036                         if (old_ce)
3037                                 iounmap(old_ce);
3038
3039                         ret = 0;
3040                         if (devfn < 0x80)
3041                                 old_ce_phys = root_entry_lctp(&re);
3042                         else
3043                                 old_ce_phys = root_entry_uctp(&re);
3044
3045                         if (!old_ce_phys) {
3046                                 if (ext && devfn == 0) {
3047                                         /* No LCTP, try UCTP */
3048                                         devfn = 0x7f;
3049                                         continue;
3050                                 } else {
3051                                         goto out;
3052                                 }
3053                         }
3054
3055                         ret = -ENOMEM;
3056                         old_ce = memremap(old_ce_phys, PAGE_SIZE,
3057                                         MEMREMAP_WB);
3058                         if (!old_ce)
3059                                 goto out;
3060
3061                         new_ce = alloc_pgtable_page(iommu->node);
3062                         if (!new_ce)
3063                                 goto out_unmap;
3064
3065                         ret = 0;
3066                 }
3067
3068                 /* Now copy the context entry */
3069                 memcpy(&ce, old_ce + idx, sizeof(ce));
3070
3071                 if (!__context_present(&ce))
3072                         continue;
3073
3074                 did = context_domain_id(&ce);
3075                 if (did >= 0 && did < cap_ndoms(iommu->cap))
3076                         set_bit(did, iommu->domain_ids);
3077
3078                 /*
3079                  * We need a marker for copied context entries. This
3080                  * marker needs to work for the old format as well as
3081                  * for extended context entries.
3082                  *
3083                  * Bit 67 of the context entry is used. In the old
3084                  * format this bit is available to software, in the
3085                  * extended format it is the PGE bit, but PGE is ignored
3086                  * by HW if PASIDs are disabled (and thus still
3087                  * available).
3088                  *
3089                  * So disable PASIDs first and then mark the entry
3090                  * copied. This means that we don't copy PASID
3091                  * translations from the old kernel, but this is fine as
3092                  * faults there are not fatal.
3093                  */
3094                 context_clear_pasid_enable(&ce);
3095                 context_set_copied(&ce);
3096
3097                 new_ce[idx] = ce;
3098         }
3099
3100         tbl[tbl_idx + pos] = new_ce;
3101
3102         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3103
3104 out_unmap:
3105         memunmap(old_ce);
3106
3107 out:
3108         return ret;
3109 }
3110
3111 static int copy_translation_tables(struct intel_iommu *iommu)
3112 {
3113         struct context_entry **ctxt_tbls;
3114         struct root_entry *old_rt;
3115         phys_addr_t old_rt_phys;
3116         int ctxt_table_entries;
3117         unsigned long flags;
3118         u64 rtaddr_reg;
3119         int bus, ret;
3120         bool new_ext, ext;
3121
3122         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3123         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
3124         new_ext    = !!ecap_ecs(iommu->ecap);
3125
3126         /*
3127          * The RTT bit can only be changed when translation is disabled,
3128          * but disabling translation means to open a window for data
3129          * corruption. So bail out and don't copy anything if we would
3130          * have to change the bit.
3131          */
3132         if (new_ext != ext)
3133                 return -EINVAL;
3134
3135         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3136         if (!old_rt_phys)
3137                 return -EINVAL;
3138
3139         old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3140         if (!old_rt)
3141                 return -ENOMEM;
3142
3143         /* This is too big for the stack - allocate it from slab */
3144         ctxt_table_entries = ext ? 512 : 256;
3145         ret = -ENOMEM;
3146         ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3147         if (!ctxt_tbls)
3148                 goto out_unmap;
3149
3150         for (bus = 0; bus < 256; bus++) {
3151                 ret = copy_context_table(iommu, &old_rt[bus],
3152                                          ctxt_tbls, bus, ext);
3153                 if (ret) {
3154                         pr_err("%s: Failed to copy context table for bus %d\n",
3155                                 iommu->name, bus);
3156                         continue;
3157                 }
3158         }
3159
3160         spin_lock_irqsave(&iommu->lock, flags);
3161
3162         /* Context tables are copied, now write them to the root_entry table */
3163         for (bus = 0; bus < 256; bus++) {
3164                 int idx = ext ? bus * 2 : bus;
3165                 u64 val;
3166
3167                 if (ctxt_tbls[idx]) {
3168                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
3169                         iommu->root_entry[bus].lo = val;
3170                 }
3171
3172                 if (!ext || !ctxt_tbls[idx + 1])
3173                         continue;
3174
3175                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3176                 iommu->root_entry[bus].hi = val;
3177         }
3178
3179         spin_unlock_irqrestore(&iommu->lock, flags);
3180
3181         kfree(ctxt_tbls);
3182
3183         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3184
3185         ret = 0;
3186
3187 out_unmap:
3188         memunmap(old_rt);
3189
3190         return ret;
3191 }
3192
3193 static int __init init_dmars(void)
3194 {
3195         struct dmar_drhd_unit *drhd;
3196         struct dmar_rmrr_unit *rmrr;
3197         bool copied_tables = false;
3198         struct device *dev;
3199         struct intel_iommu *iommu;
3200         int i, ret, cpu;
3201
3202         /*
3203          * for each drhd
3204          *    allocate root
3205          *    initialize and program root entry to not present
3206          * endfor
3207          */
3208         for_each_drhd_unit(drhd) {
3209                 /*
3210                  * lock not needed as this is only incremented in the single
3211                  * threaded kernel __init code path all other access are read
3212                  * only
3213                  */
3214                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3215                         g_num_of_iommus++;
3216                         continue;
3217                 }
3218                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3219         }
3220
3221         /* Preallocate enough resources for IOMMU hot-addition */
3222         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3223                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3224
3225         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3226                         GFP_KERNEL);
3227         if (!g_iommus) {
3228                 pr_err("Allocating global iommu array failed\n");
3229                 ret = -ENOMEM;
3230                 goto error;
3231         }
3232
3233         for_each_possible_cpu(cpu) {
3234                 struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
3235                                                               cpu);
3236
3237                 dfd->tables = kzalloc(g_num_of_iommus *
3238                                       sizeof(struct deferred_flush_table),
3239                                       GFP_KERNEL);
3240                 if (!dfd->tables) {
3241                         ret = -ENOMEM;
3242                         goto free_g_iommus;
3243                 }
3244
3245                 spin_lock_init(&dfd->lock);
3246                 setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
3247         }
3248
3249         for_each_active_iommu(iommu, drhd) {
3250                 g_iommus[iommu->seq_id] = iommu;
3251
3252                 intel_iommu_init_qi(iommu);
3253
3254                 ret = iommu_init_domains(iommu);
3255                 if (ret)
3256                         goto free_iommu;
3257
3258                 init_translation_status(iommu);
3259
3260                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3261                         iommu_disable_translation(iommu);
3262                         clear_translation_pre_enabled(iommu);
3263                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3264                                 iommu->name);
3265                 }
3266
3267                 /*
3268                  * TBD:
3269                  * we could share the same root & context tables
3270                  * among all IOMMU's. Need to Split it later.
3271                  */
3272                 ret = iommu_alloc_root_entry(iommu);
3273                 if (ret)
3274                         goto free_iommu;
3275
3276                 if (translation_pre_enabled(iommu)) {
3277                         pr_info("Translation already enabled - trying to copy translation structures\n");
3278
3279                         ret = copy_translation_tables(iommu);
3280                         if (ret) {
3281                                 /*
3282                                  * We found the IOMMU with translation
3283                                  * enabled - but failed to copy over the
3284                                  * old root-entry table. Try to proceed
3285                                  * by disabling translation now and
3286                                  * allocating a clean root-entry table.
3287                                  * This might cause DMAR faults, but
3288                                  * probably the dump will still succeed.
3289                                  */
3290                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3291                                        iommu->name);
3292                                 iommu_disable_translation(iommu);
3293                                 clear_translation_pre_enabled(iommu);
3294                         } else {
3295                                 pr_info("Copied translation tables from previous kernel for %s\n",
3296                                         iommu->name);
3297                                 copied_tables = true;
3298                         }
3299                 }
3300
3301                 if (!ecap_pass_through(iommu->ecap))
3302                         hw_pass_through = 0;
3303 #ifdef CONFIG_INTEL_IOMMU_SVM
3304                 if (pasid_enabled(iommu))
3305                         intel_svm_alloc_pasid_tables(iommu);
3306 #endif
3307         }
3308
3309         /*
3310          * Now that qi is enabled on all iommus, set the root entry and flush
3311          * caches. This is required on some Intel X58 chipsets, otherwise the
3312          * flush_context function will loop forever and the boot hangs.
3313          */
3314         for_each_active_iommu(iommu, drhd) {
3315                 iommu_flush_write_buffer(iommu);
3316                 iommu_set_root_entry(iommu);
3317                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3318                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3319         }
3320
3321         if (iommu_pass_through)
3322                 iommu_identity_mapping |= IDENTMAP_ALL;
3323
3324 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3325         iommu_identity_mapping |= IDENTMAP_GFX;
3326 #endif
3327
3328         if (iommu_identity_mapping) {
3329                 ret = si_domain_init(hw_pass_through);
3330                 if (ret)
3331                         goto free_iommu;
3332         }
3333
3334         check_tylersburg_isoch();
3335
3336         /*
3337          * If we copied translations from a previous kernel in the kdump
3338          * case, we can not assign the devices to domains now, as that
3339          * would eliminate the old mappings. So skip this part and defer
3340          * the assignment to device driver initialization time.
3341          */
3342         if (copied_tables)
3343                 goto domains_done;
3344
3345         /*
3346          * If pass through is not set or not enabled, setup context entries for
3347          * identity mappings for rmrr, gfx, and isa and may fall back to static
3348          * identity mapping if iommu_identity_mapping is set.
3349          */
3350         if (iommu_identity_mapping) {
3351                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3352                 if (ret) {
3353                         pr_crit("Failed to setup IOMMU pass-through\n");
3354                         goto free_iommu;
3355                 }
3356         }
3357         /*
3358          * For each rmrr
3359          *   for each dev attached to rmrr
3360          *   do
3361          *     locate drhd for dev, alloc domain for dev
3362          *     allocate free domain
3363          *     allocate page table entries for rmrr
3364          *     if context not allocated for bus
3365          *           allocate and init context
3366          *           set present in root table for this bus
3367          *     init context with domain, translation etc
3368          *    endfor
3369          * endfor
3370          */
3371         pr_info("Setting RMRR:\n");
3372         for_each_rmrr_units(rmrr) {
3373                 /* some BIOS lists non-exist devices in DMAR table. */
3374                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3375                                           i, dev) {
3376                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
3377                         if (ret)
3378                                 pr_err("Mapping reserved region failed\n");
3379                 }
3380         }
3381
3382         iommu_prepare_isa();
3383
3384 domains_done:
3385
3386         /*
3387          * for each drhd
3388          *   enable fault log
3389          *   global invalidate context cache
3390          *   global invalidate iotlb
3391          *   enable translation
3392          */
3393         for_each_iommu(iommu, drhd) {
3394                 if (drhd->ignored) {
3395                         /*
3396                          * we always have to disable PMRs or DMA may fail on
3397                          * this device
3398                          */
3399                         if (force_on)
3400                                 iommu_disable_protect_mem_regions(iommu);
3401                         continue;
3402                 }
3403
3404                 iommu_flush_write_buffer(iommu);
3405
3406 #ifdef CONFIG_INTEL_IOMMU_SVM
3407                 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3408                         ret = intel_svm_enable_prq(iommu);
3409                         if (ret)
3410                                 goto free_iommu;
3411                 }
3412 #endif
3413                 ret = dmar_set_interrupt(iommu);
3414                 if (ret)
3415                         goto free_iommu;
3416
3417                 if (!translation_pre_enabled(iommu))
3418                         iommu_enable_translation(iommu);
3419
3420                 iommu_disable_protect_mem_regions(iommu);
3421         }
3422
3423         return 0;
3424
3425 free_iommu:
3426         for_each_active_iommu(iommu, drhd) {
3427                 disable_dmar_iommu(iommu);
3428                 free_dmar_iommu(iommu);
3429         }
3430 free_g_iommus:
3431         for_each_possible_cpu(cpu)
3432                 kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
3433         kfree(g_iommus);
3434 error:
3435         return ret;
3436 }
3437
3438 /* This takes a number of _MM_ pages, not VTD pages */
3439 static unsigned long intel_alloc_iova(struct device *dev,
3440                                      struct dmar_domain *domain,
3441                                      unsigned long nrpages, uint64_t dma_mask)
3442 {
3443         unsigned long iova_pfn = 0;
3444
3445         /* Restrict dma_mask to the width that the iommu can handle */
3446         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3447         /* Ensure we reserve the whole size-aligned region */
3448         nrpages = __roundup_pow_of_two(nrpages);
3449
3450         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3451                 /*
3452                  * First try to allocate an io virtual address in
3453                  * DMA_BIT_MASK(32) and if that fails then try allocating
3454                  * from higher range
3455                  */
3456                 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3457                                            IOVA_PFN(DMA_BIT_MASK(32)));
3458                 if (iova_pfn)
3459                         return iova_pfn;
3460         }
3461         iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask));
3462         if (unlikely(!iova_pfn)) {
3463                 pr_err("Allocating %ld-page iova for %s failed",
3464                        nrpages, dev_name(dev));
3465                 return 0;
3466         }
3467
3468         return iova_pfn;
3469 }
3470
3471 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3472 {
3473         struct dmar_domain *domain, *tmp;
3474         struct dmar_rmrr_unit *rmrr;
3475         struct device *i_dev;
3476         int i, ret;
3477
3478         domain = find_domain(dev);
3479         if (domain)
3480                 goto out;
3481
3482         domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3483         if (!domain)
3484                 goto out;
3485
3486         /* We have a new domain - setup possible RMRRs for the device */
3487         rcu_read_lock();
3488         for_each_rmrr_units(rmrr) {
3489                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3490                                           i, i_dev) {
3491                         if (i_dev != dev)
3492                                 continue;
3493
3494                         ret = domain_prepare_identity_map(dev, domain,
3495                                                           rmrr->base_address,
3496                                                           rmrr->end_address);
3497                         if (ret)
3498                                 dev_err(dev, "Mapping reserved region failed\n");
3499                 }
3500         }
3501         rcu_read_unlock();
3502
3503         tmp = set_domain_for_dev(dev, domain);
3504         if (!tmp || domain != tmp) {
3505                 domain_exit(domain);
3506                 domain = tmp;
3507         }
3508
3509 out:
3510
3511         if (!domain)
3512                 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3513
3514
3515         return domain;
3516 }
3517
3518 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3519 {
3520         struct device_domain_info *info;
3521
3522         /* No lock here, assumes no domain exit in normal case */
3523         info = dev->archdata.iommu;
3524         if (likely(info))
3525                 return info->domain;
3526
3527         return __get_valid_domain_for_dev(dev);
3528 }
3529
3530 /* Check if the dev needs to go through non-identity map and unmap process.*/
3531 static int iommu_no_mapping(struct device *dev)
3532 {
3533         int found;
3534
3535         if (iommu_dummy(dev))
3536                 return 1;
3537
3538         if (!iommu_identity_mapping)
3539                 return 0;
3540
3541         found = identity_mapping(dev);
3542         if (found) {
3543                 if (iommu_should_identity_map(dev, 0))
3544                         return 1;
3545                 else {
3546                         /*
3547                          * 32 bit DMA is removed from si_domain and fall back
3548                          * to non-identity mapping.
3549                          */
3550                         dmar_remove_one_dev_info(si_domain, dev);
3551                         pr_info("32bit %s uses non-identity mapping\n",
3552                                 dev_name(dev));
3553                         return 0;
3554                 }
3555         } else {
3556                 /*
3557                  * In case of a detached 64 bit DMA device from vm, the device
3558                  * is put into si_domain for identity mapping.
3559                  */
3560                 if (iommu_should_identity_map(dev, 0)) {
3561                         int ret;
3562                         ret = domain_add_dev_info(si_domain, dev);
3563                         if (!ret) {
3564                                 pr_info("64bit %s uses identity mapping\n",
3565                                         dev_name(dev));
3566                                 return 1;
3567                         }
3568                 }
3569         }
3570
3571         return 0;
3572 }
3573
3574 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3575                                      size_t size, int dir, u64 dma_mask)
3576 {
3577         struct dmar_domain *domain;
3578         phys_addr_t start_paddr;
3579         unsigned long iova_pfn;
3580         int prot = 0;
3581         int ret;
3582         struct intel_iommu *iommu;
3583         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3584
3585         BUG_ON(dir == DMA_NONE);
3586
3587         if (iommu_no_mapping(dev))
3588                 return paddr;
3589
3590         domain = get_valid_domain_for_dev(dev);
3591         if (!domain)
3592                 return 0;
3593
3594         iommu = domain_get_iommu(domain);
3595         size = aligned_nrpages(paddr, size);
3596
3597         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3598         if (!iova_pfn)
3599                 goto error;
3600
3601         /*
3602          * Check if DMAR supports zero-length reads on write only
3603          * mappings..
3604          */
3605         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3606                         !cap_zlr(iommu->cap))
3607                 prot |= DMA_PTE_READ;
3608         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3609                 prot |= DMA_PTE_WRITE;
3610         /*
3611          * paddr - (paddr + size) might be partial page, we should map the whole
3612          * page.  Note: if two part of one page are separately mapped, we
3613          * might have two guest_addr mapping to the same host paddr, but this
3614          * is not a big problem
3615          */
3616         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3617                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3618         if (ret)
3619                 goto error;
3620
3621         /* it's a non-present to present mapping. Only flush if caching mode */
3622         if (cap_caching_mode(iommu->cap))
3623                 iommu_flush_iotlb_psi(iommu, domain,
3624                                       mm_to_dma_pfn(iova_pfn),
3625                                       size, 0, 1);
3626         else
3627                 iommu_flush_write_buffer(iommu);
3628
3629         start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3630         start_paddr += paddr & ~PAGE_MASK;
3631         return start_paddr;
3632
3633 error:
3634         if (iova_pfn)
3635                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3636         pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3637                 dev_name(dev), size, (unsigned long long)paddr, dir);
3638         return 0;
3639 }
3640
3641 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3642                                  unsigned long offset, size_t size,
3643                                  enum dma_data_direction dir,
3644                                  unsigned long attrs)
3645 {
3646         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3647                                   dir, *dev->dma_mask);
3648 }
3649
3650 static void flush_unmaps(struct deferred_flush_data *flush_data)
3651 {
3652         int i, j;
3653
3654         flush_data->timer_on = 0;
3655
3656         /* just flush them all */
3657         for (i = 0; i < g_num_of_iommus; i++) {
3658                 struct intel_iommu *iommu = g_iommus[i];
3659                 struct deferred_flush_table *flush_table =
3660                                 &flush_data->tables[i];
3661                 if (!iommu)
3662                         continue;
3663
3664                 if (!flush_table->next)
3665                         continue;
3666
3667                 /* In caching mode, global flushes turn emulation expensive */
3668                 if (!cap_caching_mode(iommu->cap))
3669                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3670                                          DMA_TLB_GLOBAL_FLUSH);
3671                 for (j = 0; j < flush_table->next; j++) {
3672                         unsigned long mask;
3673                         struct deferred_flush_entry *entry =
3674                                                 &flush_table->entries[j];
3675                         unsigned long iova_pfn = entry->iova_pfn;
3676                         unsigned long nrpages = entry->nrpages;
3677                         struct dmar_domain *domain = entry->domain;
3678                         struct page *freelist = entry->freelist;
3679
3680                         /* On real hardware multiple invalidations are expensive */
3681                         if (cap_caching_mode(iommu->cap))
3682                                 iommu_flush_iotlb_psi(iommu, domain,
3683                                         mm_to_dma_pfn(iova_pfn),
3684                                         nrpages, !freelist, 0);
3685                         else {
3686                                 mask = ilog2(nrpages);
3687                                 iommu_flush_dev_iotlb(domain,
3688                                                 (uint64_t)iova_pfn << PAGE_SHIFT, mask);
3689                         }
3690                         free_iova_fast(&domain->iovad, iova_pfn, nrpages);
3691                         if (freelist)
3692                                 dma_free_pagelist(freelist);
3693                 }
3694                 flush_table->next = 0;
3695         }
3696
3697         flush_data->size = 0;
3698 }
3699
3700 static void flush_unmaps_timeout(unsigned long cpuid)
3701 {
3702         struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3703         unsigned long flags;
3704
3705         spin_lock_irqsave(&flush_data->lock, flags);
3706         flush_unmaps(flush_data);
3707         spin_unlock_irqrestore(&flush_data->lock, flags);
3708 }
3709
3710 static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
3711                       unsigned long nrpages, struct page *freelist)
3712 {
3713         unsigned long flags;
3714         int entry_id, iommu_id;
3715         struct intel_iommu *iommu;
3716         struct deferred_flush_entry *entry;
3717         struct deferred_flush_data *flush_data;
3718         unsigned int cpuid;
3719
3720         cpuid = get_cpu();
3721         flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3722
3723         /* Flush all CPUs' entries to avoid deferring too much.  If
3724          * this becomes a bottleneck, can just flush us, and rely on
3725          * flush timer for the rest.
3726          */
3727         if (flush_data->size == HIGH_WATER_MARK) {
3728                 int cpu;
3729
3730                 for_each_online_cpu(cpu)
3731                         flush_unmaps_timeout(cpu);
3732         }
3733
3734         spin_lock_irqsave(&flush_data->lock, flags);
3735
3736         iommu = domain_get_iommu(dom);
3737         iommu_id = iommu->seq_id;
3738
3739         entry_id = flush_data->tables[iommu_id].next;
3740         ++(flush_data->tables[iommu_id].next);
3741
3742         entry = &flush_data->tables[iommu_id].entries[entry_id];
3743         entry->domain = dom;
3744         entry->iova_pfn = iova_pfn;
3745         entry->nrpages = nrpages;
3746         entry->freelist = freelist;
3747
3748         if (!flush_data->timer_on) {
3749                 mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
3750                 flush_data->timer_on = 1;
3751         }
3752         flush_data->size++;
3753         spin_unlock_irqrestore(&flush_data->lock, flags);
3754
3755         put_cpu();
3756 }
3757
3758 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3759 {
3760         struct dmar_domain *domain;
3761         unsigned long start_pfn, last_pfn;
3762         unsigned long nrpages;
3763         unsigned long iova_pfn;
3764         struct intel_iommu *iommu;
3765         struct page *freelist;
3766
3767         if (iommu_no_mapping(dev))
3768                 return;
3769
3770         domain = find_domain(dev);
3771         BUG_ON(!domain);
3772
3773         iommu = domain_get_iommu(domain);
3774
3775         iova_pfn = IOVA_PFN(dev_addr);
3776
3777         nrpages = aligned_nrpages(dev_addr, size);
3778         start_pfn = mm_to_dma_pfn(iova_pfn);
3779         last_pfn = start_pfn + nrpages - 1;
3780
3781         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3782                  dev_name(dev), start_pfn, last_pfn);
3783
3784         freelist = domain_unmap(domain, start_pfn, last_pfn);
3785
3786         if (intel_iommu_strict) {
3787                 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3788                                       nrpages, !freelist, 0);
3789                 /* free iova */
3790                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3791                 dma_free_pagelist(freelist);
3792         } else {
3793                 add_unmap(domain, iova_pfn, nrpages, freelist);
3794                 /*
3795                  * queue up the release of the unmap to save the 1/6th of the
3796                  * cpu used up by the iotlb flush operation...
3797                  */
3798         }
3799 }
3800
3801 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3802                              size_t size, enum dma_data_direction dir,
3803                              unsigned long attrs)
3804 {
3805         intel_unmap(dev, dev_addr, size);
3806 }
3807
3808 static void *intel_alloc_coherent(struct device *dev, size_t size,
3809                                   dma_addr_t *dma_handle, gfp_t flags,
3810                                   unsigned long attrs)
3811 {
3812         struct page *page = NULL;
3813         int order;
3814
3815         size = PAGE_ALIGN(size);
3816         order = get_order(size);
3817
3818         if (!iommu_no_mapping(dev))
3819                 flags &= ~(GFP_DMA | GFP_DMA32);
3820         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3821                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3822                         flags |= GFP_DMA;
3823                 else
3824                         flags |= GFP_DMA32;
3825         }
3826
3827         if (gfpflags_allow_blocking(flags)) {
3828                 unsigned int count = size >> PAGE_SHIFT;
3829
3830                 page = dma_alloc_from_contiguous(dev, count, order);
3831                 if (page && iommu_no_mapping(dev) &&
3832                     page_to_phys(page) + size > dev->coherent_dma_mask) {
3833                         dma_release_from_contiguous(dev, page, count);
3834                         page = NULL;
3835                 }
3836         }
3837
3838         if (!page)
3839                 page = alloc_pages(flags, order);
3840         if (!page)
3841                 return NULL;
3842         memset(page_address(page), 0, size);
3843
3844         *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3845                                          DMA_BIDIRECTIONAL,
3846                                          dev->coherent_dma_mask);
3847         if (*dma_handle)
3848                 return page_address(page);
3849         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3850                 __free_pages(page, order);
3851
3852         return NULL;
3853 }
3854
3855 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3856                                 dma_addr_t dma_handle, unsigned long attrs)
3857 {
3858         int order;
3859         struct page *page = virt_to_page(vaddr);
3860
3861         size = PAGE_ALIGN(size);
3862         order = get_order(size);
3863
3864         intel_unmap(dev, dma_handle, size);
3865         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3866                 __free_pages(page, order);
3867 }
3868
3869 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3870                            int nelems, enum dma_data_direction dir,
3871                            unsigned long attrs)
3872 {
3873         dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3874         unsigned long nrpages = 0;
3875         struct scatterlist *sg;
3876         int i;
3877
3878         for_each_sg(sglist, sg, nelems, i) {
3879                 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3880         }
3881
3882         intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3883 }
3884
3885 static int intel_nontranslate_map_sg(struct device *hddev,
3886         struct scatterlist *sglist, int nelems, int dir)
3887 {
3888         int i;
3889         struct scatterlist *sg;
3890
3891         for_each_sg(sglist, sg, nelems, i) {
3892                 BUG_ON(!sg_page(sg));
3893                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3894                 sg->dma_length = sg->length;
3895         }
3896         return nelems;
3897 }
3898
3899 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3900                         enum dma_data_direction dir, unsigned long attrs)
3901 {
3902         int i;
3903         struct dmar_domain *domain;
3904         size_t size = 0;
3905         int prot = 0;
3906         unsigned long iova_pfn;
3907         int ret;
3908         struct scatterlist *sg;
3909         unsigned long start_vpfn;
3910         struct intel_iommu *iommu;
3911
3912         BUG_ON(dir == DMA_NONE);
3913         if (iommu_no_mapping(dev))
3914                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3915
3916         domain = get_valid_domain_for_dev(dev);
3917         if (!domain)
3918                 return 0;
3919
3920         iommu = domain_get_iommu(domain);
3921
3922         for_each_sg(sglist, sg, nelems, i)
3923                 size += aligned_nrpages(sg->offset, sg->length);
3924
3925         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3926                                 *dev->dma_mask);
3927         if (!iova_pfn) {
3928                 sglist->dma_length = 0;
3929                 return 0;
3930         }
3931
3932         /*
3933          * Check if DMAR supports zero-length reads on write only
3934          * mappings..
3935          */
3936         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3937                         !cap_zlr(iommu->cap))
3938                 prot |= DMA_PTE_READ;
3939         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3940                 prot |= DMA_PTE_WRITE;
3941
3942         start_vpfn = mm_to_dma_pfn(iova_pfn);
3943
3944         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3945         if (unlikely(ret)) {
3946                 dma_pte_free_pagetable(domain, start_vpfn,
3947                                        start_vpfn + size - 1);
3948                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3949                 return 0;
3950         }
3951
3952         /* it's a non-present to present mapping. Only flush if caching mode */
3953         if (cap_caching_mode(iommu->cap))
3954                 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3955         else
3956                 iommu_flush_write_buffer(iommu);
3957
3958         return nelems;
3959 }
3960
3961 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3962 {
3963         return !dma_addr;
3964 }
3965
3966 struct dma_map_ops intel_dma_ops = {
3967         .alloc = intel_alloc_coherent,
3968         .free = intel_free_coherent,
3969         .map_sg = intel_map_sg,
3970         .unmap_sg = intel_unmap_sg,
3971         .map_page = intel_map_page,
3972         .unmap_page = intel_unmap_page,
3973         .mapping_error = intel_mapping_error,
3974 };
3975
3976 static inline int iommu_domain_cache_init(void)
3977 {
3978         int ret = 0;
3979
3980         iommu_domain_cache = kmem_cache_create("iommu_domain",
3981                                          sizeof(struct dmar_domain),
3982                                          0,
3983                                          SLAB_HWCACHE_ALIGN,
3984
3985                                          NULL);
3986         if (!iommu_domain_cache) {
3987                 pr_err("Couldn't create iommu_domain cache\n");
3988                 ret = -ENOMEM;
3989         }
3990
3991         return ret;
3992 }
3993
3994 static inline int iommu_devinfo_cache_init(void)
3995 {
3996         int ret = 0;
3997
3998         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3999                                          sizeof(struct device_domain_info),
4000                                          0,
4001                                          SLAB_HWCACHE_ALIGN,
4002                                          NULL);
4003         if (!iommu_devinfo_cache) {
4004                 pr_err("Couldn't create devinfo cache\n");
4005                 ret = -ENOMEM;
4006         }
4007
4008         return ret;
4009 }
4010
4011 static int __init iommu_init_mempool(void)
4012 {
4013         int ret;
4014         ret = iova_cache_get();
4015         if (ret)
4016                 return ret;
4017
4018         ret = iommu_domain_cache_init();
4019         if (ret)
4020                 goto domain_error;
4021
4022         ret = iommu_devinfo_cache_init();
4023         if (!ret)
4024                 return ret;
4025
4026         kmem_cache_destroy(iommu_domain_cache);
4027 domain_error:
4028         iova_cache_put();
4029
4030         return -ENOMEM;
4031 }
4032
4033 static void __init iommu_exit_mempool(void)
4034 {
4035         kmem_cache_destroy(iommu_devinfo_cache);
4036         kmem_cache_destroy(iommu_domain_cache);
4037         iova_cache_put();
4038 }
4039
4040 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
4041 {
4042         struct dmar_drhd_unit *drhd;
4043         u32 vtbar;
4044         int rc;
4045
4046         /* We know that this device on this chipset has its own IOMMU.
4047          * If we find it under a different IOMMU, then the BIOS is lying
4048          * to us. Hope that the IOMMU for this device is actually
4049          * disabled, and it needs no translation...
4050          */
4051         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4052         if (rc) {
4053                 /* "can't" happen */
4054                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4055                 return;
4056         }
4057         vtbar &= 0xffff0000;
4058
4059         /* we know that the this iommu should be at offset 0xa000 from vtbar */
4060         drhd = dmar_find_matched_drhd_unit(pdev);
4061         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
4062                             TAINT_FIRMWARE_WORKAROUND,
4063                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
4064                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4065 }
4066 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4067
4068 static void __init init_no_remapping_devices(void)
4069 {
4070         struct dmar_drhd_unit *drhd;
4071         struct device *dev;
4072         int i;
4073
4074         for_each_drhd_unit(drhd) {
4075                 if (!drhd->include_all) {
4076                         for_each_active_dev_scope(drhd->devices,
4077                                                   drhd->devices_cnt, i, dev)
4078                                 break;
4079                         /* ignore DMAR unit if no devices exist */
4080                         if (i == drhd->devices_cnt)
4081                                 drhd->ignored = 1;
4082                 }
4083         }
4084
4085         for_each_active_drhd_unit(drhd) {
4086                 if (drhd->include_all)
4087                         continue;
4088
4089                 for_each_active_dev_scope(drhd->devices,
4090                                           drhd->devices_cnt, i, dev)
4091                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4092                                 break;
4093                 if (i < drhd->devices_cnt)
4094                         continue;
4095
4096                 /* This IOMMU has *only* gfx devices. Either bypass it or
4097                    set the gfx_mapped flag, as appropriate */
4098                 if (dmar_map_gfx) {
4099                         intel_iommu_gfx_mapped = 1;
4100                 } else {
4101                         drhd->ignored = 1;
4102                         for_each_active_dev_scope(drhd->devices,
4103                                                   drhd->devices_cnt, i, dev)
4104                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4105                 }
4106         }
4107 }
4108
4109 #ifdef CONFIG_SUSPEND
4110 static int init_iommu_hw(void)
4111 {
4112         struct dmar_drhd_unit *drhd;
4113         struct intel_iommu *iommu = NULL;
4114
4115         for_each_active_iommu(iommu, drhd)
4116                 if (iommu->qi)
4117                         dmar_reenable_qi(iommu);
4118
4119         for_each_iommu(iommu, drhd) {
4120                 if (drhd->ignored) {
4121                         /*
4122                          * we always have to disable PMRs or DMA may fail on
4123                          * this device
4124                          */
4125                         if (force_on)
4126                                 iommu_disable_protect_mem_regions(iommu);
4127                         continue;
4128                 }
4129
4130                 iommu_flush_write_buffer(iommu);
4131
4132                 iommu_set_root_entry(iommu);
4133
4134                 iommu->flush.flush_context(iommu, 0, 0, 0,
4135                                            DMA_CCMD_GLOBAL_INVL);
4136                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4137                 iommu_enable_translation(iommu);
4138                 iommu_disable_protect_mem_regions(iommu);
4139         }
4140
4141         return 0;
4142 }
4143
4144 static void iommu_flush_all(void)
4145 {
4146         struct dmar_drhd_unit *drhd;
4147         struct intel_iommu *iommu;
4148
4149         for_each_active_iommu(iommu, drhd) {
4150                 iommu->flush.flush_context(iommu, 0, 0, 0,
4151                                            DMA_CCMD_GLOBAL_INVL);
4152                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4153                                          DMA_TLB_GLOBAL_FLUSH);
4154         }
4155 }
4156
4157 static int iommu_suspend(void)
4158 {
4159         struct dmar_drhd_unit *drhd;
4160         struct intel_iommu *iommu = NULL;
4161         unsigned long flag;
4162
4163         for_each_active_iommu(iommu, drhd) {
4164                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4165                                                  GFP_ATOMIC);
4166                 if (!iommu->iommu_state)
4167                         goto nomem;
4168         }
4169
4170         iommu_flush_all();
4171
4172         for_each_active_iommu(iommu, drhd) {
4173                 iommu_disable_translation(iommu);
4174
4175                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4176
4177                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4178                         readl(iommu->reg + DMAR_FECTL_REG);
4179                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4180                         readl(iommu->reg + DMAR_FEDATA_REG);
4181                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4182                         readl(iommu->reg + DMAR_FEADDR_REG);
4183                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4184                         readl(iommu->reg + DMAR_FEUADDR_REG);
4185
4186                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4187         }
4188         return 0;
4189
4190 nomem:
4191         for_each_active_iommu(iommu, drhd)
4192                 kfree(iommu->iommu_state);
4193
4194         return -ENOMEM;
4195 }
4196
4197 static void iommu_resume(void)
4198 {
4199         struct dmar_drhd_unit *drhd;
4200         struct intel_iommu *iommu = NULL;
4201         unsigned long flag;
4202
4203         if (init_iommu_hw()) {
4204                 if (force_on)
4205                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4206                 else
4207                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4208                 return;
4209         }
4210
4211         for_each_active_iommu(iommu, drhd) {
4212
4213                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4214
4215                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4216                         iommu->reg + DMAR_FECTL_REG);
4217                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4218                         iommu->reg + DMAR_FEDATA_REG);
4219                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4220                         iommu->reg + DMAR_FEADDR_REG);
4221                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4222                         iommu->reg + DMAR_FEUADDR_REG);
4223
4224                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4225         }
4226
4227         for_each_active_iommu(iommu, drhd)
4228                 kfree(iommu->iommu_state);
4229 }
4230
4231 static struct syscore_ops iommu_syscore_ops = {
4232         .resume         = iommu_resume,
4233         .suspend        = iommu_suspend,
4234 };
4235
4236 static void __init init_iommu_pm_ops(void)
4237 {
4238         register_syscore_ops(&iommu_syscore_ops);
4239 }
4240
4241 #else
4242 static inline void init_iommu_pm_ops(void) {}
4243 #endif  /* CONFIG_PM */
4244
4245
4246 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4247 {
4248         struct acpi_dmar_reserved_memory *rmrr;
4249         struct dmar_rmrr_unit *rmrru;
4250
4251         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4252         if (!rmrru)
4253                 return -ENOMEM;
4254
4255         rmrru->hdr = header;
4256         rmrr = (struct acpi_dmar_reserved_memory *)header;
4257         rmrru->base_address = rmrr->base_address;
4258         rmrru->end_address = rmrr->end_address;
4259         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4260                                 ((void *)rmrr) + rmrr->header.length,
4261                                 &rmrru->devices_cnt);
4262         if (rmrru->devices_cnt && rmrru->devices == NULL) {
4263                 kfree(rmrru);
4264                 return -ENOMEM;
4265         }
4266
4267         list_add(&rmrru->list, &dmar_rmrr_units);
4268
4269         return 0;
4270 }
4271
4272 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4273 {
4274         struct dmar_atsr_unit *atsru;
4275         struct acpi_dmar_atsr *tmp;
4276
4277         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4278                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4279                 if (atsr->segment != tmp->segment)
4280                         continue;
4281                 if (atsr->header.length != tmp->header.length)
4282                         continue;
4283                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4284                         return atsru;
4285         }
4286
4287         return NULL;
4288 }
4289
4290 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4291 {
4292         struct acpi_dmar_atsr *atsr;
4293         struct dmar_atsr_unit *atsru;
4294
4295         if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4296                 return 0;
4297
4298         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4299         atsru = dmar_find_atsr(atsr);
4300         if (atsru)
4301                 return 0;
4302
4303         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4304         if (!atsru)
4305                 return -ENOMEM;
4306
4307         /*
4308          * If memory is allocated from slab by ACPI _DSM method, we need to
4309          * copy the memory content because the memory buffer will be freed
4310          * on return.
4311          */
4312         atsru->hdr = (void *)(atsru + 1);
4313         memcpy(atsru->hdr, hdr, hdr->length);
4314         atsru->include_all = atsr->flags & 0x1;
4315         if (!atsru->include_all) {
4316                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4317                                 (void *)atsr + atsr->header.length,
4318                                 &atsru->devices_cnt);
4319                 if (atsru->devices_cnt && atsru->devices == NULL) {
4320                         kfree(atsru);
4321                         return -ENOMEM;
4322                 }
4323         }
4324
4325         list_add_rcu(&atsru->list, &dmar_atsr_units);
4326
4327         return 0;
4328 }
4329
4330 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4331 {
4332         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4333         kfree(atsru);
4334 }
4335
4336 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4337 {
4338         struct acpi_dmar_atsr *atsr;
4339         struct dmar_atsr_unit *atsru;
4340
4341         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4342         atsru = dmar_find_atsr(atsr);
4343         if (atsru) {
4344                 list_del_rcu(&atsru->list);
4345                 synchronize_rcu();
4346                 intel_iommu_free_atsr(atsru);
4347         }
4348
4349         return 0;
4350 }
4351
4352 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4353 {
4354         int i;
4355         struct device *dev;
4356         struct acpi_dmar_atsr *atsr;
4357         struct dmar_atsr_unit *atsru;
4358
4359         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4360         atsru = dmar_find_atsr(atsr);
4361         if (!atsru)
4362                 return 0;
4363
4364         if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4365                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4366                                           i, dev)
4367                         return -EBUSY;
4368         }
4369
4370         return 0;
4371 }
4372
4373 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4374 {
4375         int sp, ret = 0;
4376         struct intel_iommu *iommu = dmaru->iommu;
4377
4378         if (g_iommus[iommu->seq_id])
4379                 return 0;
4380
4381         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4382                 pr_warn("%s: Doesn't support hardware pass through.\n",
4383                         iommu->name);
4384                 return -ENXIO;
4385         }
4386         if (!ecap_sc_support(iommu->ecap) &&
4387             domain_update_iommu_snooping(iommu)) {
4388                 pr_warn("%s: Doesn't support snooping.\n",
4389                         iommu->name);
4390                 return -ENXIO;
4391         }
4392         sp = domain_update_iommu_superpage(iommu) - 1;
4393         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4394                 pr_warn("%s: Doesn't support large page.\n",
4395                         iommu->name);
4396                 return -ENXIO;
4397         }
4398
4399         /*
4400          * Disable translation if already enabled prior to OS handover.
4401          */
4402         if (iommu->gcmd & DMA_GCMD_TE)
4403                 iommu_disable_translation(iommu);
4404
4405         g_iommus[iommu->seq_id] = iommu;
4406         ret = iommu_init_domains(iommu);
4407         if (ret == 0)
4408                 ret = iommu_alloc_root_entry(iommu);
4409         if (ret)
4410                 goto out;
4411
4412 #ifdef CONFIG_INTEL_IOMMU_SVM
4413         if (pasid_enabled(iommu))
4414                 intel_svm_alloc_pasid_tables(iommu);
4415 #endif
4416
4417         if (dmaru->ignored) {
4418                 /*
4419                  * we always have to disable PMRs or DMA may fail on this device
4420                  */
4421                 if (force_on)
4422                         iommu_disable_protect_mem_regions(iommu);
4423                 return 0;
4424         }
4425
4426         intel_iommu_init_qi(iommu);
4427         iommu_flush_write_buffer(iommu);
4428
4429 #ifdef CONFIG_INTEL_IOMMU_SVM
4430         if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4431                 ret = intel_svm_enable_prq(iommu);
4432                 if (ret)
4433                         goto disable_iommu;
4434         }
4435 #endif
4436         ret = dmar_set_interrupt(iommu);
4437         if (ret)
4438                 goto disable_iommu;
4439
4440         iommu_set_root_entry(iommu);
4441         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4442         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4443         iommu_enable_translation(iommu);
4444
4445         iommu_disable_protect_mem_regions(iommu);
4446         return 0;
4447
4448 disable_iommu:
4449         disable_dmar_iommu(iommu);
4450 out:
4451         free_dmar_iommu(iommu);
4452         return ret;
4453 }
4454
4455 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4456 {
4457         int ret = 0;
4458         struct intel_iommu *iommu = dmaru->iommu;
4459
4460         if (!intel_iommu_enabled)
4461                 return 0;
4462         if (iommu == NULL)
4463                 return -EINVAL;
4464
4465         if (insert) {
4466                 ret = intel_iommu_add(dmaru);
4467         } else {
4468                 disable_dmar_iommu(iommu);
4469                 free_dmar_iommu(iommu);
4470         }
4471
4472         return ret;
4473 }
4474
4475 static void intel_iommu_free_dmars(void)
4476 {
4477         struct dmar_rmrr_unit *rmrru, *rmrr_n;
4478         struct dmar_atsr_unit *atsru, *atsr_n;
4479
4480         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4481                 list_del(&rmrru->list);
4482                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4483                 kfree(rmrru);
4484         }
4485
4486         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4487                 list_del(&atsru->list);
4488                 intel_iommu_free_atsr(atsru);
4489         }
4490 }
4491
4492 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4493 {
4494         int i, ret = 1;
4495         struct pci_bus *bus;
4496         struct pci_dev *bridge = NULL;
4497         struct device *tmp;
4498         struct acpi_dmar_atsr *atsr;
4499         struct dmar_atsr_unit *atsru;
4500
4501         dev = pci_physfn(dev);
4502         for (bus = dev->bus; bus; bus = bus->parent) {
4503                 bridge = bus->self;
4504                 /* If it's an integrated device, allow ATS */
4505                 if (!bridge)
4506                         return 1;
4507                 /* Connected via non-PCIe: no ATS */
4508                 if (!pci_is_pcie(bridge) ||
4509                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4510                         return 0;
4511                 /* If we found the root port, look it up in the ATSR */
4512                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4513                         break;
4514         }
4515
4516         rcu_read_lock();
4517         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4518                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4519                 if (atsr->segment != pci_domain_nr(dev->bus))
4520                         continue;
4521
4522                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4523                         if (tmp == &bridge->dev)
4524                                 goto out;
4525
4526                 if (atsru->include_all)
4527                         goto out;
4528         }
4529         ret = 0;
4530 out:
4531         rcu_read_unlock();
4532
4533         return ret;
4534 }
4535
4536 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4537 {
4538         int ret = 0;
4539         struct dmar_rmrr_unit *rmrru;
4540         struct dmar_atsr_unit *atsru;
4541         struct acpi_dmar_atsr *atsr;
4542         struct acpi_dmar_reserved_memory *rmrr;
4543
4544         if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4545                 return 0;
4546
4547         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4548                 rmrr = container_of(rmrru->hdr,
4549                                     struct acpi_dmar_reserved_memory, header);
4550                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4551                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4552                                 ((void *)rmrr) + rmrr->header.length,
4553                                 rmrr->segment, rmrru->devices,
4554                                 rmrru->devices_cnt);
4555                         if(ret < 0)
4556                                 return ret;
4557                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4558                         dmar_remove_dev_scope(info, rmrr->segment,
4559                                 rmrru->devices, rmrru->devices_cnt);
4560                 }
4561         }
4562
4563         list_for_each_entry(atsru, &dmar_atsr_units, list) {
4564                 if (atsru->include_all)
4565                         continue;
4566
4567                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4568                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4569                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4570                                         (void *)atsr + atsr->header.length,
4571                                         atsr->segment, atsru->devices,
4572                                         atsru->devices_cnt);
4573                         if (ret > 0)
4574                                 break;
4575                         else if(ret < 0)
4576                                 return ret;
4577                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4578                         if (dmar_remove_dev_scope(info, atsr->segment,
4579                                         atsru->devices, atsru->devices_cnt))
4580                                 break;
4581                 }
4582         }
4583
4584         return 0;
4585 }
4586
4587 /*
4588  * Here we only respond to action of unbound device from driver.
4589  *
4590  * Added device is not attached to its DMAR domain here yet. That will happen
4591  * when mapping the device to iova.
4592  */
4593 static int device_notifier(struct notifier_block *nb,
4594                                   unsigned long action, void *data)
4595 {
4596         struct device *dev = data;
4597         struct dmar_domain *domain;
4598
4599         if (iommu_dummy(dev))
4600                 return 0;
4601
4602         if (action != BUS_NOTIFY_REMOVED_DEVICE)
4603                 return 0;
4604
4605         domain = find_domain(dev);
4606         if (!domain)
4607                 return 0;
4608
4609         dmar_remove_one_dev_info(domain, dev);
4610         if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4611                 domain_exit(domain);
4612
4613         return 0;
4614 }
4615
4616 static struct notifier_block device_nb = {
4617         .notifier_call = device_notifier,
4618 };
4619
4620 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4621                                        unsigned long val, void *v)
4622 {
4623         struct memory_notify *mhp = v;
4624         unsigned long long start, end;
4625         unsigned long start_vpfn, last_vpfn;
4626
4627         switch (val) {
4628         case MEM_GOING_ONLINE:
4629                 start = mhp->start_pfn << PAGE_SHIFT;
4630                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4631                 if (iommu_domain_identity_map(si_domain, start, end)) {
4632                         pr_warn("Failed to build identity map for [%llx-%llx]\n",
4633                                 start, end);
4634                         return NOTIFY_BAD;
4635                 }
4636                 break;
4637
4638         case MEM_OFFLINE:
4639         case MEM_CANCEL_ONLINE:
4640                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4641                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4642                 while (start_vpfn <= last_vpfn) {
4643                         struct iova *iova;
4644                         struct dmar_drhd_unit *drhd;
4645                         struct intel_iommu *iommu;
4646                         struct page *freelist;
4647
4648                         iova = find_iova(&si_domain->iovad, start_vpfn);
4649                         if (iova == NULL) {
4650                                 pr_debug("Failed get IOVA for PFN %lx\n",
4651                                          start_vpfn);
4652                                 break;
4653                         }
4654
4655                         iova = split_and_remove_iova(&si_domain->iovad, iova,
4656                                                      start_vpfn, last_vpfn);
4657                         if (iova == NULL) {
4658                                 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4659                                         start_vpfn, last_vpfn);
4660                                 return NOTIFY_BAD;
4661                         }
4662
4663                         freelist = domain_unmap(si_domain, iova->pfn_lo,
4664                                                iova->pfn_hi);
4665
4666                         rcu_read_lock();
4667                         for_each_active_iommu(iommu, drhd)
4668                                 iommu_flush_iotlb_psi(iommu, si_domain,
4669                                         iova->pfn_lo, iova_size(iova),
4670                                         !freelist, 0);
4671                         rcu_read_unlock();
4672                         dma_free_pagelist(freelist);
4673
4674                         start_vpfn = iova->pfn_hi + 1;
4675                         free_iova_mem(iova);
4676                 }
4677                 break;
4678         }
4679
4680         return NOTIFY_OK;
4681 }
4682
4683 static struct notifier_block intel_iommu_memory_nb = {
4684         .notifier_call = intel_iommu_memory_notifier,
4685         .priority = 0
4686 };
4687
4688 static void free_all_cpu_cached_iovas(unsigned int cpu)
4689 {
4690         int i;
4691
4692         for (i = 0; i < g_num_of_iommus; i++) {
4693                 struct intel_iommu *iommu = g_iommus[i];
4694                 struct dmar_domain *domain;
4695                 int did;
4696
4697                 if (!iommu)
4698                         continue;
4699
4700                 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4701                         domain = get_iommu_domain(iommu, (u16)did);
4702
4703                         if (!domain)
4704                                 continue;
4705                         free_cpu_cached_iovas(cpu, &domain->iovad);
4706                 }
4707         }
4708 }
4709
4710 static int intel_iommu_cpu_dead(unsigned int cpu)
4711 {
4712         free_all_cpu_cached_iovas(cpu);
4713         flush_unmaps_timeout(cpu);
4714         return 0;
4715 }
4716
4717 static ssize_t intel_iommu_show_version(struct device *dev,
4718                                         struct device_attribute *attr,
4719                                         char *buf)
4720 {
4721         struct intel_iommu *iommu = dev_get_drvdata(dev);
4722         u32 ver = readl(iommu->reg + DMAR_VER_REG);
4723         return sprintf(buf, "%d:%d\n",
4724                        DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4725 }
4726 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4727
4728 static ssize_t intel_iommu_show_address(struct device *dev,
4729                                         struct device_attribute *attr,
4730                                         char *buf)
4731 {
4732         struct intel_iommu *iommu = dev_get_drvdata(dev);
4733         return sprintf(buf, "%llx\n", iommu->reg_phys);
4734 }
4735 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4736
4737 static ssize_t intel_iommu_show_cap(struct device *dev,
4738                                     struct device_attribute *attr,
4739                                     char *buf)
4740 {
4741         struct intel_iommu *iommu = dev_get_drvdata(dev);
4742         return sprintf(buf, "%llx\n", iommu->cap);
4743 }
4744 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4745
4746 static ssize_t intel_iommu_show_ecap(struct device *dev,
4747                                     struct device_attribute *attr,
4748                                     char *buf)
4749 {
4750         struct intel_iommu *iommu = dev_get_drvdata(dev);
4751         return sprintf(buf, "%llx\n", iommu->ecap);
4752 }
4753 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4754
4755 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4756                                       struct device_attribute *attr,
4757                                       char *buf)
4758 {
4759         struct intel_iommu *iommu = dev_get_drvdata(dev);
4760         return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4761 }
4762 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4763
4764 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4765                                            struct device_attribute *attr,
4766                                            char *buf)
4767 {
4768         struct intel_iommu *iommu = dev_get_drvdata(dev);
4769         return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4770                                                   cap_ndoms(iommu->cap)));
4771 }
4772 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4773
4774 static struct attribute *intel_iommu_attrs[] = {
4775         &dev_attr_version.attr,
4776         &dev_attr_address.attr,
4777         &dev_attr_cap.attr,
4778         &dev_attr_ecap.attr,
4779         &dev_attr_domains_supported.attr,
4780         &dev_attr_domains_used.attr,
4781         NULL,
4782 };
4783
4784 static struct attribute_group intel_iommu_group = {
4785         .name = "intel-iommu",
4786         .attrs = intel_iommu_attrs,
4787 };
4788
4789 const struct attribute_group *intel_iommu_groups[] = {
4790         &intel_iommu_group,
4791         NULL,
4792 };
4793
4794 int __init intel_iommu_init(void)
4795 {
4796         int ret = -ENODEV;
4797         struct dmar_drhd_unit *drhd;
4798         struct intel_iommu *iommu;
4799
4800         /* VT-d is required for a TXT/tboot launch, so enforce that */
4801         force_on = tboot_force_iommu();
4802
4803         if (iommu_init_mempool()) {
4804                 if (force_on)
4805                         panic("tboot: Failed to initialize iommu memory\n");
4806                 return -ENOMEM;
4807         }
4808
4809         down_write(&dmar_global_lock);
4810         if (dmar_table_init()) {
4811                 if (force_on)
4812                         panic("tboot: Failed to initialize DMAR table\n");
4813                 goto out_free_dmar;
4814         }
4815
4816         if (dmar_dev_scope_init() < 0) {
4817                 if (force_on)
4818                         panic("tboot: Failed to initialize DMAR device scope\n");
4819                 goto out_free_dmar;
4820         }
4821
4822         if (no_iommu || dmar_disabled)
4823                 goto out_free_dmar;
4824
4825         if (list_empty(&dmar_rmrr_units))
4826                 pr_info("No RMRR found\n");
4827
4828         if (list_empty(&dmar_atsr_units))
4829                 pr_info("No ATSR found\n");
4830
4831         if (dmar_init_reserved_ranges()) {
4832                 if (force_on)
4833                         panic("tboot: Failed to reserve iommu ranges\n");
4834                 goto out_free_reserved_range;
4835         }
4836
4837         init_no_remapping_devices();
4838
4839         ret = init_dmars();
4840         if (ret) {
4841                 if (force_on)
4842                         panic("tboot: Failed to initialize DMARs\n");
4843                 pr_err("Initialization failed\n");
4844                 goto out_free_reserved_range;
4845         }
4846         up_write(&dmar_global_lock);
4847         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4848
4849 #ifdef CONFIG_SWIOTLB
4850         swiotlb = 0;
4851 #endif
4852         dma_ops = &intel_dma_ops;
4853
4854         init_iommu_pm_ops();
4855
4856         for_each_active_iommu(iommu, drhd)
4857                 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4858                                                        intel_iommu_groups,
4859                                                        "%s", iommu->name);
4860
4861         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4862         bus_register_notifier(&pci_bus_type, &device_nb);
4863         if (si_domain && !hw_pass_through)
4864                 register_memory_notifier(&intel_iommu_memory_nb);
4865         cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4866                           intel_iommu_cpu_dead);
4867         intel_iommu_enabled = 1;
4868
4869         return 0;
4870
4871 out_free_reserved_range:
4872         put_iova_domain(&reserved_iova_list);
4873 out_free_dmar:
4874         intel_iommu_free_dmars();
4875         up_write(&dmar_global_lock);
4876         iommu_exit_mempool();
4877         return ret;
4878 }
4879
4880 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4881 {
4882         struct intel_iommu *iommu = opaque;
4883
4884         domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4885         return 0;
4886 }
4887
4888 /*
4889  * NB - intel-iommu lacks any sort of reference counting for the users of
4890  * dependent devices.  If multiple endpoints have intersecting dependent
4891  * devices, unbinding the driver from any one of them will possibly leave
4892  * the others unable to operate.
4893  */
4894 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4895 {
4896         if (!iommu || !dev || !dev_is_pci(dev))
4897                 return;
4898
4899         pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4900 }
4901
4902 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4903 {
4904         struct intel_iommu *iommu;
4905         unsigned long flags;
4906
4907         assert_spin_locked(&device_domain_lock);
4908
4909         if (WARN_ON(!info))
4910                 return;
4911
4912         iommu = info->iommu;
4913
4914         if (info->dev) {
4915                 iommu_disable_dev_iotlb(info);
4916                 domain_context_clear(iommu, info->dev);
4917         }
4918
4919         unlink_domain_info(info);
4920
4921         spin_lock_irqsave(&iommu->lock, flags);
4922         domain_detach_iommu(info->domain, iommu);
4923         spin_unlock_irqrestore(&iommu->lock, flags);
4924
4925         free_devinfo_mem(info);
4926 }
4927
4928 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4929                                      struct device *dev)
4930 {
4931         struct device_domain_info *info;
4932         unsigned long flags;
4933
4934         spin_lock_irqsave(&device_domain_lock, flags);
4935         info = dev->archdata.iommu;
4936         __dmar_remove_one_dev_info(info);
4937         spin_unlock_irqrestore(&device_domain_lock, flags);
4938 }
4939
4940 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4941 {
4942         int adjust_width;
4943
4944         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4945                         DMA_32BIT_PFN);
4946         domain_reserve_special_ranges(domain);
4947
4948         /* calculate AGAW */
4949         domain->gaw = guest_width;
4950         adjust_width = guestwidth_to_adjustwidth(guest_width);
4951         domain->agaw = width_to_agaw(adjust_width);
4952
4953         domain->iommu_coherency = 0;
4954         domain->iommu_snooping = 0;
4955         domain->iommu_superpage = 0;
4956         domain->max_addr = 0;
4957
4958         /* always allocate the top pgd */
4959         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4960         if (!domain->pgd)
4961                 return -ENOMEM;
4962         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4963         return 0;
4964 }
4965
4966 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4967 {
4968         struct dmar_domain *dmar_domain;
4969         struct iommu_domain *domain;
4970
4971         if (type != IOMMU_DOMAIN_UNMANAGED)
4972                 return NULL;
4973
4974         dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4975         if (!dmar_domain) {
4976                 pr_err("Can't allocate dmar_domain\n");
4977                 return NULL;
4978         }
4979         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4980                 pr_err("Domain initialization failed\n");
4981                 domain_exit(dmar_domain);
4982                 return NULL;
4983         }
4984         domain_update_iommu_cap(dmar_domain);
4985
4986         domain = &dmar_domain->domain;
4987         domain->geometry.aperture_start = 0;
4988         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4989         domain->geometry.force_aperture = true;
4990
4991         return domain;
4992 }
4993
4994 static void intel_iommu_domain_free(struct iommu_domain *domain)
4995 {
4996         domain_exit(to_dmar_domain(domain));
4997 }
4998
4999 static int intel_iommu_attach_device(struct iommu_domain *domain,
5000                                      struct device *dev)
5001 {
5002         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5003         struct intel_iommu *iommu;
5004         int addr_width;
5005         u8 bus, devfn;
5006
5007         if (device_is_rmrr_locked(dev)) {
5008                 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement.  Contact your platform vendor.\n");
5009                 return -EPERM;
5010         }
5011
5012         /* normally dev is not mapped */
5013         if (unlikely(domain_context_mapped(dev))) {
5014                 struct dmar_domain *old_domain;
5015
5016                 old_domain = find_domain(dev);
5017                 if (old_domain) {
5018                         rcu_read_lock();
5019                         dmar_remove_one_dev_info(old_domain, dev);
5020                         rcu_read_unlock();
5021
5022                         if (!domain_type_is_vm_or_si(old_domain) &&
5023                              list_empty(&old_domain->devices))
5024                                 domain_exit(old_domain);
5025                 }
5026         }
5027
5028         iommu = device_to_iommu(dev, &bus, &devfn);
5029         if (!iommu)
5030                 return -ENODEV;
5031
5032         /* check if this iommu agaw is sufficient for max mapped address */
5033         addr_width = agaw_to_width(iommu->agaw);
5034         if (addr_width > cap_mgaw(iommu->cap))
5035                 addr_width = cap_mgaw(iommu->cap);
5036
5037         if (dmar_domain->max_addr > (1LL << addr_width)) {
5038                 pr_err("%s: iommu width (%d) is not "
5039                        "sufficient for the mapped address (%llx)\n",
5040                        __func__, addr_width, dmar_domain->max_addr);
5041                 return -EFAULT;
5042         }
5043         dmar_domain->gaw = addr_width;
5044
5045         /*
5046          * Knock out extra levels of page tables if necessary
5047          */
5048         while (iommu->agaw < dmar_domain->agaw) {
5049                 struct dma_pte *pte;
5050
5051                 pte = dmar_domain->pgd;
5052                 if (dma_pte_present(pte)) {
5053                         dmar_domain->pgd = (struct dma_pte *)
5054                                 phys_to_virt(dma_pte_addr(pte));
5055                         free_pgtable_page(pte);
5056                 }
5057                 dmar_domain->agaw--;
5058         }
5059
5060         return domain_add_dev_info(dmar_domain, dev);
5061 }
5062
5063 static void intel_iommu_detach_device(struct iommu_domain *domain,
5064                                       struct device *dev)
5065 {
5066         dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5067 }
5068
5069 static int intel_iommu_map(struct iommu_domain *domain,
5070                            unsigned long iova, phys_addr_t hpa,
5071                            size_t size, int iommu_prot)
5072 {
5073         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5074         u64 max_addr;
5075         int prot = 0;
5076         int ret;
5077
5078         if (iommu_prot & IOMMU_READ)
5079                 prot |= DMA_PTE_READ;
5080         if (iommu_prot & IOMMU_WRITE)
5081                 prot |= DMA_PTE_WRITE;
5082         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5083                 prot |= DMA_PTE_SNP;
5084
5085         max_addr = iova + size;
5086         if (dmar_domain->max_addr < max_addr) {
5087                 u64 end;
5088
5089                 /* check if minimum agaw is sufficient for mapped address */
5090                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5091                 if (end < max_addr) {
5092                         pr_err("%s: iommu width (%d) is not "
5093                                "sufficient for the mapped address (%llx)\n",
5094                                __func__, dmar_domain->gaw, max_addr);
5095                         return -EFAULT;
5096                 }
5097                 dmar_domain->max_addr = max_addr;
5098         }
5099         /* Round up size to next multiple of PAGE_SIZE, if it and
5100            the low bits of hpa would take us onto the next page */
5101         size = aligned_nrpages(hpa, size);
5102         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5103                                  hpa >> VTD_PAGE_SHIFT, size, prot);
5104         return ret;
5105 }
5106
5107 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5108                                 unsigned long iova, size_t size)
5109 {
5110         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5111         struct page *freelist = NULL;
5112         struct intel_iommu *iommu;
5113         unsigned long start_pfn, last_pfn;
5114         unsigned int npages;
5115         int iommu_id, level = 0;
5116
5117         /* Cope with horrid API which requires us to unmap more than the
5118            size argument if it happens to be a large-page mapping. */
5119         BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5120
5121         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5122                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5123
5124         start_pfn = iova >> VTD_PAGE_SHIFT;
5125         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5126
5127         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5128
5129         npages = last_pfn - start_pfn + 1;
5130
5131         for_each_domain_iommu(iommu_id, dmar_domain) {
5132                 iommu = g_iommus[iommu_id];
5133
5134                 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5135                                       start_pfn, npages, !freelist, 0);
5136         }
5137
5138         dma_free_pagelist(freelist);
5139
5140         if (dmar_domain->max_addr == iova + size)
5141                 dmar_domain->max_addr = iova;
5142
5143         return size;
5144 }
5145
5146 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5147                                             dma_addr_t iova)
5148 {
5149         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5150         struct dma_pte *pte;
5151         int level = 0;
5152         u64 phys = 0;
5153
5154         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5155         if (pte)
5156                 phys = dma_pte_addr(pte);
5157
5158         return phys;
5159 }
5160
5161 static bool intel_iommu_capable(enum iommu_cap cap)
5162 {
5163         if (cap == IOMMU_CAP_CACHE_COHERENCY)
5164                 return domain_update_iommu_snooping(NULL) == 1;
5165         if (cap == IOMMU_CAP_INTR_REMAP)
5166                 return irq_remapping_enabled == 1;
5167
5168         return false;
5169 }
5170
5171 static int intel_iommu_add_device(struct device *dev)
5172 {
5173         struct intel_iommu *iommu;
5174         struct iommu_group *group;
5175         u8 bus, devfn;
5176
5177         iommu = device_to_iommu(dev, &bus, &devfn);
5178         if (!iommu)
5179                 return -ENODEV;
5180
5181         iommu_device_link(iommu->iommu_dev, dev);
5182
5183         group = iommu_group_get_for_dev(dev);
5184
5185         if (IS_ERR(group))
5186                 return PTR_ERR(group);
5187
5188         iommu_group_put(group);
5189         return 0;
5190 }
5191
5192 static void intel_iommu_remove_device(struct device *dev)
5193 {
5194         struct intel_iommu *iommu;
5195         u8 bus, devfn;
5196
5197         iommu = device_to_iommu(dev, &bus, &devfn);
5198         if (!iommu)
5199                 return;
5200
5201         iommu_group_remove_device(dev);
5202
5203         iommu_device_unlink(iommu->iommu_dev, dev);
5204 }
5205
5206 #ifdef CONFIG_INTEL_IOMMU_SVM
5207 #define MAX_NR_PASID_BITS (20)
5208 static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu)
5209 {
5210         /*
5211          * Convert ecap_pss to extend context entry pts encoding, also
5212          * respect the soft pasid_max value set by the iommu.
5213          * - number of PASID bits = ecap_pss + 1
5214          * - number of PASID table entries = 2^(pts + 5)
5215          * Therefore, pts = ecap_pss - 4
5216          * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15
5217          */
5218         if (ecap_pss(iommu->ecap) < 5)
5219                 return 0;
5220
5221         /* pasid_max is encoded as actual number of entries not the bits */
5222         return find_first_bit((unsigned long *)&iommu->pasid_max,
5223                         MAX_NR_PASID_BITS) - 5;
5224 }
5225
5226 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5227 {
5228         struct device_domain_info *info;
5229         struct context_entry *context;
5230         struct dmar_domain *domain;
5231         unsigned long flags;
5232         u64 ctx_lo;
5233         int ret;
5234
5235         domain = get_valid_domain_for_dev(sdev->dev);
5236         if (!domain)
5237                 return -EINVAL;
5238
5239         spin_lock_irqsave(&device_domain_lock, flags);
5240         spin_lock(&iommu->lock);
5241
5242         ret = -EINVAL;
5243         info = sdev->dev->archdata.iommu;
5244         if (!info || !info->pasid_supported)
5245                 goto out;
5246
5247         context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5248         if (WARN_ON(!context))
5249                 goto out;
5250
5251         ctx_lo = context[0].lo;
5252
5253         sdev->did = domain->iommu_did[iommu->seq_id];
5254         sdev->sid = PCI_DEVID(info->bus, info->devfn);
5255
5256         if (!(ctx_lo & CONTEXT_PASIDE)) {
5257                 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5258                 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
5259                         intel_iommu_get_pts(iommu);
5260
5261                 wmb();
5262                 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5263                  * extended to permit requests-with-PASID if the PASIDE bit
5264                  * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5265                  * however, the PASIDE bit is ignored and requests-with-PASID
5266                  * are unconditionally blocked. Which makes less sense.
5267                  * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5268                  * "guest mode" translation types depending on whether ATS
5269                  * is available or not. Annoyingly, we can't use the new
5270                  * modes *unless* PASIDE is set. */
5271                 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5272                         ctx_lo &= ~CONTEXT_TT_MASK;
5273                         if (info->ats_supported)
5274                                 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5275                         else
5276                                 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5277                 }
5278                 ctx_lo |= CONTEXT_PASIDE;
5279                 if (iommu->pasid_state_table)
5280                         ctx_lo |= CONTEXT_DINVE;
5281                 if (info->pri_supported)
5282                         ctx_lo |= CONTEXT_PRS;
5283                 context[0].lo = ctx_lo;
5284                 wmb();
5285                 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5286                                            DMA_CCMD_MASK_NOBIT,
5287                                            DMA_CCMD_DEVICE_INVL);
5288         }
5289
5290         /* Enable PASID support in the device, if it wasn't already */
5291         if (!info->pasid_enabled)
5292                 iommu_enable_dev_iotlb(info);
5293
5294         if (info->ats_enabled) {
5295                 sdev->dev_iotlb = 1;
5296                 sdev->qdep = info->ats_qdep;
5297                 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5298                         sdev->qdep = 0;
5299         }
5300         ret = 0;
5301
5302  out:
5303         spin_unlock(&iommu->lock);
5304         spin_unlock_irqrestore(&device_domain_lock, flags);
5305
5306         return ret;
5307 }
5308
5309 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5310 {
5311         struct intel_iommu *iommu;
5312         u8 bus, devfn;
5313
5314         if (iommu_dummy(dev)) {
5315                 dev_warn(dev,
5316                          "No IOMMU translation for device; cannot enable SVM\n");
5317                 return NULL;
5318         }
5319
5320         iommu = device_to_iommu(dev, &bus, &devfn);
5321         if ((!iommu)) {
5322                 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5323                 return NULL;
5324         }
5325
5326         if (!iommu->pasid_table) {
5327                 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5328                 return NULL;
5329         }
5330
5331         return iommu;
5332 }
5333 #endif /* CONFIG_INTEL_IOMMU_SVM */
5334
5335 static const struct iommu_ops intel_iommu_ops = {
5336         .capable        = intel_iommu_capable,
5337         .domain_alloc   = intel_iommu_domain_alloc,
5338         .domain_free    = intel_iommu_domain_free,
5339         .attach_dev     = intel_iommu_attach_device,
5340         .detach_dev     = intel_iommu_detach_device,
5341         .map            = intel_iommu_map,
5342         .unmap          = intel_iommu_unmap,
5343         .map_sg         = default_iommu_map_sg,
5344         .iova_to_phys   = intel_iommu_iova_to_phys,
5345         .add_device     = intel_iommu_add_device,
5346         .remove_device  = intel_iommu_remove_device,
5347         .device_group   = pci_device_group,
5348         .pgsize_bitmap  = INTEL_IOMMU_PGSIZES,
5349 };
5350
5351 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5352 {
5353         /* G4x/GM45 integrated gfx dmar support is totally busted. */
5354         pr_info("Disabling IOMMU for graphics on this chipset\n");
5355         dmar_map_gfx = 0;
5356 }
5357
5358 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5359 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5360 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5361 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5362 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5363 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5364 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5365
5366 static void quirk_iommu_rwbf(struct pci_dev *dev)
5367 {
5368         /*
5369          * Mobile 4 Series Chipset neglects to set RWBF capability,
5370          * but needs it. Same seems to hold for the desktop versions.
5371          */
5372         pr_info("Forcing write-buffer flush capability\n");
5373         rwbf_quirk = 1;
5374 }
5375
5376 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5377 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5378 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5379 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5380 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5381 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5382 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5383
5384 #define GGC 0x52
5385 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
5386 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
5387 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
5388 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
5389 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
5390 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
5391 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
5392 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
5393
5394 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5395 {
5396         unsigned short ggc;
5397
5398         if (pci_read_config_word(dev, GGC, &ggc))
5399                 return;
5400
5401         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5402                 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5403                 dmar_map_gfx = 0;
5404         } else if (dmar_map_gfx) {
5405                 /* we have to ensure the gfx device is idle before we flush */
5406                 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5407                 intel_iommu_strict = 1;
5408        }
5409 }
5410 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5411 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5412 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5413 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5414
5415 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5416    ISOCH DMAR unit for the Azalia sound device, but not give it any
5417    TLB entries, which causes it to deadlock. Check for that.  We do
5418    this in a function called from init_dmars(), instead of in a PCI
5419    quirk, because we don't want to print the obnoxious "BIOS broken"
5420    message if VT-d is actually disabled.
5421 */
5422 static void __init check_tylersburg_isoch(void)
5423 {
5424         struct pci_dev *pdev;
5425         uint32_t vtisochctrl;
5426
5427         /* If there's no Azalia in the system anyway, forget it. */
5428         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5429         if (!pdev)
5430                 return;
5431         pci_dev_put(pdev);
5432
5433         /* System Management Registers. Might be hidden, in which case
5434            we can't do the sanity check. But that's OK, because the
5435            known-broken BIOSes _don't_ actually hide it, so far. */
5436         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5437         if (!pdev)
5438                 return;
5439
5440         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5441                 pci_dev_put(pdev);
5442                 return;
5443         }
5444
5445         pci_dev_put(pdev);
5446
5447         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5448         if (vtisochctrl & 1)
5449                 return;
5450
5451         /* Drop all bits other than the number of TLB entries */
5452         vtisochctrl &= 0x1c;
5453
5454         /* If we have the recommended number of TLB entries (16), fine. */
5455         if (vtisochctrl == 0x10)
5456                 return;
5457
5458         /* Zero TLB entries? You get to ride the short bus to school. */
5459         if (!vtisochctrl) {
5460                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5461                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5462                      dmi_get_system_info(DMI_BIOS_VENDOR),
5463                      dmi_get_system_info(DMI_BIOS_VERSION),
5464                      dmi_get_system_info(DMI_PRODUCT_VERSION));
5465                 iommu_identity_mapping |= IDENTMAP_AZALIA;
5466                 return;
5467         }
5468
5469         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5470                vtisochctrl);
5471 }