drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  */
  19
  20 #include <linux/init.h>
  21 #include <linux/bitmap.h>
  22 #include <linux/debugfs.h>
  23 #include <linux/export.h>
  24 #include <linux/slab.h>
  25 #include <linux/irq.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/spinlock.h>
  28 #include <linux/pci.h>
  29 #include <linux/dmar.h>
  30 #include <linux/dma-mapping.h>
  31 #include <linux/mempool.h>
  32 #include <linux/memory.h>
  33 #include <linux/timer.h>
  34 #include <linux/iova.h>
  35 #include <linux/iommu.h>
  36 #include <linux/intel-iommu.h>
  37 #include <linux/syscore_ops.h>
  38 #include <linux/tboot.h>
  39 #include <linux/dmi.h>
  40 #include <linux/pci-ats.h>
  41 #include <linux/memblock.h>
  42 #include <asm/irq_remapping.h>
  43 #include <asm/cacheflush.h>
  44 #include <asm/iommu.h>
  45
  46 #include "irq_remapping.h"
  47 #include "pci.h"
  48
  49 #define ROOT_SIZE               VTD_PAGE_SIZE
  50 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  51
  52 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  53 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  54 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  55
  56 #define IOAPIC_RANGE_START      (0xfee00000)
  57 #define IOAPIC_RANGE_END        (0xfeefffff)
  58 #define IOVA_START_ADDR         (0x1000)
  59
  60 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  61
  62 #define MAX_AGAW_WIDTH 64
  63 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  64
  65 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  66 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  67
  68 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  69    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  70 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  71                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  72 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  73
  74 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  75 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  76 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  77
  78 /* page table handling */
  79 #define LEVEL_STRIDE            (9)
  80 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  81
  82 /*
  83  * This bitmap is used to advertise the page sizes our hardware support
  84  * to the IOMMU core, which will then use this information to split
  85  * physically contiguous memory regions it is mapping into page sizes
  86  * that we support.
  87  *
  88  * Traditionally the IOMMU core just handed us the mappings directly,
  89  * after making sure the size is an order of a 4KiB page and that the
  90  * mapping has natural alignment.
  91  *
  92  * To retain this behavior, we currently advertise that we support
  93  * all page sizes that are an order of 4KiB.
  94  *
  95  * If at some point we'd like to utilize the IOMMU core's new behavior,
  96  * we could change this to advertise the real page sizes we support.
  97  */
  98 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
  99
 100 static inline int agaw_to_level(int agaw)
 101 {
 102         return agaw + 2;
 103 }
 104
 105 static inline int agaw_to_width(int agaw)
 106 {
 107         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 108 }
 109
 110 static inline int width_to_agaw(int width)
 111 {
 112         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 113 }
 114
 115 static inline unsigned int level_to_offset_bits(int level)
 116 {
 117         return (level - 1) * LEVEL_STRIDE;
 118 }
 119
 120 static inline int pfn_level_offset(unsigned long pfn, int level)
 121 {
 122         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 123 }
 124
 125 static inline unsigned long level_mask(int level)
 126 {
 127         return -1UL << level_to_offset_bits(level);
 128 }
 129
 130 static inline unsigned long level_size(int level)
 131 {
 132         return 1UL << level_to_offset_bits(level);
 133 }
 134
 135 static inline unsigned long align_to_level(unsigned long pfn, int level)
 136 {
 137         return (pfn + level_size(level) - 1) & level_mask(level);
 138 }
 139
 140 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 141 {
 142         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 143 }
 144
 145 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 146    are never going to work. */
 147 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 148 {
 149         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 150 }
 151
 152 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 153 {
 154         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 155 }
 156 static inline unsigned long page_to_dma_pfn(struct page *pg)
 157 {
 158         return mm_to_dma_pfn(page_to_pfn(pg));
 159 }
 160 static inline unsigned long virt_to_dma_pfn(void *p)
 161 {
 162         return page_to_dma_pfn(virt_to_page(p));
 163 }
 164
 165 /* global iommu list, set NULL for ignored DMAR units */
 166 static struct intel_iommu **g_iommus;
 167
 168 static void __init check_tylersburg_isoch(void);
 169 static int rwbf_quirk;
 170
 171 /*
 172  * set to 1 to panic kernel if can't successfully enable VT-d
 173  * (used when kernel is launched w/ TXT)
 174  */
 175 static int force_on = 0;
 176
 177 /*
 178  * 0: Present
 179  * 1-11: Reserved
 180  * 12-63: Context Ptr (12 - (haw-1))
 181  * 64-127: Reserved
 182  */
 183 struct root_entry {
 184         u64     val;
 185         u64     rsvd1;
 186 };
 187 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 188 static inline bool root_present(struct root_entry *root)
 189 {
 190         return (root->val & 1);
 191 }
 192 static inline void set_root_present(struct root_entry *root)
 193 {
 194         root->val |= 1;
 195 }
 196 static inline void set_root_value(struct root_entry *root, unsigned long value)
 197 {
 198         root->val |= value & VTD_PAGE_MASK;
 199 }
 200
 201 static inline struct context_entry *
 202 get_context_addr_from_root(struct root_entry *root)
 203 {
 204         return (struct context_entry *)
 205                 (root_present(root)?phys_to_virt(
 206                 root->val & VTD_PAGE_MASK) :
 207                 NULL);
 208 }
 209
 210 /*
 211  * low 64 bits:
 212  * 0: present
 213  * 1: fault processing disable
 214  * 2-3: translation type
 215  * 12-63: address space root
 216  * high 64 bits:
 217  * 0-2: address width
 218  * 3-6: aval
 219  * 8-23: domain id
 220  */
 221 struct context_entry {
 222         u64 lo;
 223         u64 hi;
 224 };
 225
 226 static inline bool context_present(struct context_entry *context)
 227 {
 228         return (context->lo & 1);
 229 }
 230 static inline void context_set_present(struct context_entry *context)
 231 {
 232         context->lo |= 1;
 233 }
 234
 235 static inline void context_set_fault_enable(struct context_entry *context)
 236 {
 237         context->lo &= (((u64)-1) << 2) | 1;
 238 }
 239
 240 static inline void context_set_translation_type(struct context_entry *context,
 241                                                 unsigned long value)
 242 {
 243         context->lo &= (((u64)-1) << 4) | 3;
 244         context->lo |= (value & 3) << 2;
 245 }
 246
 247 static inline void context_set_address_root(struct context_entry *context,
 248                                             unsigned long value)
 249 {
 250         context->lo |= value & VTD_PAGE_MASK;
 251 }
 252
 253 static inline void context_set_address_width(struct context_entry *context,
 254                                              unsigned long value)
 255 {
 256         context->hi |= value & 7;
 257 }
 258
 259 static inline void context_set_domain_id(struct context_entry *context,
 260                                          unsigned long value)
 261 {
 262         context->hi |= (value & ((1 << 16) - 1)) << 8;
 263 }
 264
 265 static inline void context_clear_entry(struct context_entry *context)
 266 {
 267         context->lo = 0;
 268         context->hi = 0;
 269 }
 270
 271 /*
 272  * 0: readable
 273  * 1: writable
 274  * 2-6: reserved
 275  * 7: super page
 276  * 8-10: available
 277  * 11: snoop behavior
 278  * 12-63: Host physcial address
 279  */
 280 struct dma_pte {
 281         u64 val;
 282 };
 283
 284 static inline void dma_clear_pte(struct dma_pte *pte)
 285 {
 286         pte->val = 0;
 287 }
 288
 289 static inline u64 dma_pte_addr(struct dma_pte *pte)
 290 {
 291 #ifdef CONFIG_64BIT
 292         return pte->val & VTD_PAGE_MASK;
 293 #else
 294         /* Must have a full atomic 64-bit read */
 295         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 296 #endif
 297 }
 298
 299 static inline bool dma_pte_present(struct dma_pte *pte)
 300 {
 301         return (pte->val & 3) != 0;
 302 }
 303
 304 static inline bool dma_pte_superpage(struct dma_pte *pte)
 305 {
 306         return (pte->val & (1 << 7));
 307 }
 308
 309 static inline int first_pte_in_page(struct dma_pte *pte)
 310 {
 311         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 312 }
 313
 314 /*
 315  * This domain is a statically identity mapping domain.
 316  *      1. This domain creats a static 1:1 mapping to all usable memory.
 317  *      2. It maps to each iommu if successful.
 318  *      3. Each iommu mapps to this domain if successful.
 319  */
 320 static struct dmar_domain *si_domain;
 321 static int hw_pass_through = 1;
 322
 323 /* devices under the same p2p bridge are owned in one domain */
 324 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
 325
 326 /* domain represents a virtual machine, more than one devices
 327  * across iommus may be owned in one domain, e.g. kvm guest.
 328  */
 329 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 1)
 330
 331 /* si_domain contains mulitple devices */
 332 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 2)
 333
 334 /* define the limit of IOMMUs supported in each domain */
 335 #ifdef  CONFIG_X86
 336 # define        IOMMU_UNITS_SUPPORTED   MAX_IO_APICS
 337 #else
 338 # define        IOMMU_UNITS_SUPPORTED   64
 339 #endif
 340
 341 struct dmar_domain {
 342         int     id;                     /* domain id */
 343         int     nid;                    /* node id */
 344         DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
 345                                         /* bitmap of iommus this domain uses*/
 346
 347         struct list_head devices;       /* all devices' list */
 348         struct iova_domain iovad;       /* iova's that belong to this domain */
 349
 350         struct dma_pte  *pgd;           /* virtual address */
 351         int             gaw;            /* max guest address width */
 352
 353         /* adjusted guest address width, 0 is level 2 30-bit */
 354         int             agaw;
 355
 356         int             flags;          /* flags to find out type of domain */
 357
 358         int             iommu_coherency;/* indicate coherency of iommu access */
 359         int             iommu_snooping; /* indicate snooping control feature*/
 360         int             iommu_count;    /* reference count of iommu */
 361         int             iommu_superpage;/* Level of superpages supported:
 362                                            0 == 4KiB (no superpages), 1 == 2MiB,
 363                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 364         spinlock_t      iommu_lock;     /* protect iommu set in domain */
 365         u64             max_addr;       /* maximum mapped address */
 366 };
 367
 368 /* PCI domain-device relationship */
 369 struct device_domain_info {
 370         struct list_head link;  /* link to domain siblings */
 371         struct list_head global; /* link to global list */
 372         u8 bus;                 /* PCI bus number */
 373         u8 devfn;               /* PCI devfn number */
 374         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
 375         struct intel_iommu *iommu; /* IOMMU used by this device */
 376         struct dmar_domain *domain; /* pointer to domain */
 377 };
 378
 379 struct dmar_rmrr_unit {
 380         struct list_head list;          /* list of rmrr units   */
 381         struct acpi_dmar_header *hdr;   /* ACPI header          */
 382         u64     base_address;           /* reserved base address*/
 383         u64     end_address;            /* reserved end address */
 384         struct dmar_dev_scope *devices; /* target devices */
 385         int     devices_cnt;            /* target device count */
 386 };
 387
 388 struct dmar_atsr_unit {
 389         struct list_head list;          /* list of ATSR units */
 390         struct acpi_dmar_header *hdr;   /* ACPI header */
 391         struct dmar_dev_scope *devices; /* target devices */
 392         int devices_cnt;                /* target device count */
 393         u8 include_all:1;               /* include all ports */
 394 };
 395
 396 static LIST_HEAD(dmar_atsr_units);
 397 static LIST_HEAD(dmar_rmrr_units);
 398
 399 #define for_each_rmrr_units(rmrr) \
 400         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 401
 402 static void flush_unmaps_timeout(unsigned long data);
 403
 404 static DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 405
 406 #define HIGH_WATER_MARK 250
 407 struct deferred_flush_tables {
 408         int next;
 409         struct iova *iova[HIGH_WATER_MARK];
 410         struct dmar_domain *domain[HIGH_WATER_MARK];
 411         struct page *freelist[HIGH_WATER_MARK];
 412 };
 413
 414 static struct deferred_flush_tables *deferred_flush;
 415
 416 /* bitmap for indexing intel_iommus */
 417 static int g_num_of_iommus;
 418
 419 static DEFINE_SPINLOCK(async_umap_flush_lock);
 420 static LIST_HEAD(unmaps_to_do);
 421
 422 static int timer_on;
 423 static long list_size;
 424
 425 static void domain_exit(struct dmar_domain *domain);
 426 static void domain_remove_dev_info(struct dmar_domain *domain);
 427 static void domain_remove_one_dev_info(struct dmar_domain *domain,
 428                                        struct device *dev);
 429 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
 430                                            struct device *dev);
 431
 432 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 433 int dmar_disabled = 0;
 434 #else
 435 int dmar_disabled = 1;
 436 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 437
 438 int intel_iommu_enabled = 0;
 439 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 440
 441 static int dmar_map_gfx = 1;
 442 static int dmar_forcedac;
 443 static int intel_iommu_strict;
 444 static int intel_iommu_superpage = 1;
 445
 446 int intel_iommu_gfx_mapped;
 447 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 448
 449 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 450 static DEFINE_SPINLOCK(device_domain_lock);
 451 static LIST_HEAD(device_domain_list);
 452
 453 static struct iommu_ops intel_iommu_ops;
 454
 455 static int __init intel_iommu_setup(char *str)
 456 {
 457         if (!str)
 458                 return -EINVAL;
 459         while (*str) {
 460                 if (!strncmp(str, "on", 2)) {
 461                         dmar_disabled = 0;
 462                         printk(KERN_INFO "Intel-IOMMU: enabled\n");
 463                 } else if (!strncmp(str, "off", 3)) {
 464                         dmar_disabled = 1;
 465                         printk(KERN_INFO "Intel-IOMMU: disabled\n");
 466                 } else if (!strncmp(str, "igfx_off", 8)) {
 467                         dmar_map_gfx = 0;
 468                         printk(KERN_INFO
 469                                 "Intel-IOMMU: disable GFX device mapping\n");
 470                 } else if (!strncmp(str, "forcedac", 8)) {
 471                         printk(KERN_INFO
 472                                 "Intel-IOMMU: Forcing DAC for PCI devices\n");
 473                         dmar_forcedac = 1;
 474                 } else if (!strncmp(str, "strict", 6)) {
 475                         printk(KERN_INFO
 476                                 "Intel-IOMMU: disable batched IOTLB flush\n");
 477                         intel_iommu_strict = 1;
 478                 } else if (!strncmp(str, "sp_off", 6)) {
 479                         printk(KERN_INFO
 480                                 "Intel-IOMMU: disable supported super page\n");
 481                         intel_iommu_superpage = 0;
 482                 }
 483
 484                 str += strcspn(str, ",");
 485                 while (*str == ',')
 486                         str++;
 487         }
 488         return 0;
 489 }
 490 __setup("intel_iommu=", intel_iommu_setup);
 491
 492 static struct kmem_cache *iommu_domain_cache;
 493 static struct kmem_cache *iommu_devinfo_cache;
 494 static struct kmem_cache *iommu_iova_cache;
 495
 496 static inline void *alloc_pgtable_page(int node)
 497 {
 498         struct page *page;
 499         void *vaddr = NULL;
 500
 501         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 502         if (page)
 503                 vaddr = page_address(page);
 504         return vaddr;
 505 }
 506
 507 static inline void free_pgtable_page(void *vaddr)
 508 {
 509         free_page((unsigned long)vaddr);
 510 }
 511
 512 static inline void *alloc_domain_mem(void)
 513 {
 514         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 515 }
 516
 517 static void free_domain_mem(void *vaddr)
 518 {
 519         kmem_cache_free(iommu_domain_cache, vaddr);
 520 }
 521
 522 static inline void * alloc_devinfo_mem(void)
 523 {
 524         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 525 }
 526
 527 static inline void free_devinfo_mem(void *vaddr)
 528 {
 529         kmem_cache_free(iommu_devinfo_cache, vaddr);
 530 }
 531
 532 struct iova *alloc_iova_mem(void)
 533 {
 534         return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
 535 }
 536
 537 void free_iova_mem(struct iova *iova)
 538 {
 539         kmem_cache_free(iommu_iova_cache, iova);
 540 }
 541
 542
 543 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 544 {
 545         unsigned long sagaw;
 546         int agaw = -1;
 547
 548         sagaw = cap_sagaw(iommu->cap);
 549         for (agaw = width_to_agaw(max_gaw);
 550              agaw >= 0; agaw--) {
 551                 if (test_bit(agaw, &sagaw))
 552                         break;
 553         }
 554
 555         return agaw;
 556 }
 557
 558 /*
 559  * Calculate max SAGAW for each iommu.
 560  */
 561 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 562 {
 563         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 564 }
 565
 566 /*
 567  * calculate agaw for each iommu.
 568  * "SAGAW" may be different across iommus, use a default agaw, and
 569  * get a supported less agaw for iommus that don't support the default agaw.
 570  */
 571 int iommu_calculate_agaw(struct intel_iommu *iommu)
 572 {
 573         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 574 }
 575
 576 /* This functionin only returns single iommu in a domain */
 577 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 578 {
 579         int iommu_id;
 580
 581         /* si_domain and vm domain should not get here. */
 582         BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
 583         BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
 584
 585         iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
 586         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 587                 return NULL;
 588
 589         return g_iommus[iommu_id];
 590 }
 591
 592 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 593 {
 594         struct dmar_drhd_unit *drhd;
 595         struct intel_iommu *iommu;
 596         int i, found = 0;
 597
 598         domain->iommu_coherency = 1;
 599
 600         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
 601                 found = 1;
 602                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 603                         domain->iommu_coherency = 0;
 604                         break;
 605                 }
 606         }
 607         if (found)
 608                 return;
 609
 610         /* No hardware attached; use lowest common denominator */
 611         rcu_read_lock();
 612         for_each_active_iommu(iommu, drhd) {
 613                 if (!ecap_coherent(iommu->ecap)) {
 614                         domain->iommu_coherency = 0;
 615                         break;
 616                 }
 617         }
 618         rcu_read_unlock();
 619 }
 620
 621 static void domain_update_iommu_snooping(struct dmar_domain *domain)
 622 {
 623         int i;
 624
 625         domain->iommu_snooping = 1;
 626
 627         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
 628                 if (!ecap_sc_support(g_iommus[i]->ecap)) {
 629                         domain->iommu_snooping = 0;
 630                         break;
 631                 }
 632         }
 633 }
 634
 635 static void domain_update_iommu_superpage(struct dmar_domain *domain)
 636 {
 637         struct dmar_drhd_unit *drhd;
 638         struct intel_iommu *iommu = NULL;
 639         int mask = 0xf;
 640
 641         if (!intel_iommu_superpage) {
 642                 domain->iommu_superpage = 0;
 643                 return;
 644         }
 645
 646         /* set iommu_superpage to the smallest common denominator */
 647         rcu_read_lock();
 648         for_each_active_iommu(iommu, drhd) {
 649                 mask &= cap_super_page_val(iommu->cap);
 650                 if (!mask) {
 651                         break;
 652                 }
 653         }
 654         rcu_read_unlock();
 655
 656         domain->iommu_superpage = fls(mask);
 657 }
 658
 659 /* Some capabilities may be different across iommus */
 660 static void domain_update_iommu_cap(struct dmar_domain *domain)
 661 {
 662         domain_update_iommu_coherency(domain);
 663         domain_update_iommu_snooping(domain);
 664         domain_update_iommu_superpage(domain);
 665 }
 666
 667 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 668 {
 669         struct dmar_drhd_unit *drhd = NULL;
 670         struct intel_iommu *iommu;
 671         struct device *tmp;
 672         struct pci_dev *ptmp, *pdev = NULL;
 673         u16 segment;
 674         int i;
 675
 676         if (dev_is_pci(dev)) {
 677                 pdev = to_pci_dev(dev);
 678                 segment = pci_domain_nr(pdev->bus);
 679         } else if (ACPI_COMPANION(dev))
 680                 dev = &ACPI_COMPANION(dev)->dev;
 681
 682         rcu_read_lock();
 683         for_each_active_iommu(iommu, drhd) {
 684                 if (pdev && segment != drhd->segment)
 685                         continue;
 686
 687                 for_each_active_dev_scope(drhd->devices,
 688                                           drhd->devices_cnt, i, tmp) {
 689                         if (tmp == dev) {
 690                                 *bus = drhd->devices[i].bus;
 691                                 *devfn = drhd->devices[i].devfn;
 692                                 goto out;
 693                         }
 694
 695                         if (!pdev || !dev_is_pci(tmp))
 696                                 continue;
 697
 698                         ptmp = to_pci_dev(tmp);
 699                         if (ptmp->subordinate &&
 700                             ptmp->subordinate->number <= pdev->bus->number &&
 701                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 702                                 goto got_pdev;
 703                 }
 704
 705                 if (pdev && drhd->include_all) {
 706                 got_pdev:
 707                         *bus = pdev->bus->number;
 708                         *devfn = pdev->devfn;
 709                         goto out;
 710                 }
 711         }
 712         iommu = NULL;
 713  out:
 714         rcu_read_unlock();
 715
 716         return iommu;
 717 }
 718
 719 static void domain_flush_cache(struct dmar_domain *domain,
 720                                void *addr, int size)
 721 {
 722         if (!domain->iommu_coherency)
 723                 clflush_cache_range(addr, size);
 724 }
 725
 726 /* Gets context entry for a given bus and devfn */
 727 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 728                 u8 bus, u8 devfn)
 729 {
 730         struct root_entry *root;
 731         struct context_entry *context;
 732         unsigned long phy_addr;
 733         unsigned long flags;
 734
 735         spin_lock_irqsave(&iommu->lock, flags);
 736         root = &iommu->root_entry[bus];
 737         context = get_context_addr_from_root(root);
 738         if (!context) {
 739                 context = (struct context_entry *)
 740                                 alloc_pgtable_page(iommu->node);
 741                 if (!context) {
 742                         spin_unlock_irqrestore(&iommu->lock, flags);
 743                         return NULL;
 744                 }
 745                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 746                 phy_addr = virt_to_phys((void *)context);
 747                 set_root_value(root, phy_addr);
 748                 set_root_present(root);
 749                 __iommu_flush_cache(iommu, root, sizeof(*root));
 750         }
 751         spin_unlock_irqrestore(&iommu->lock, flags);
 752         return &context[devfn];
 753 }
 754
 755 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 756 {
 757         struct root_entry *root;
 758         struct context_entry *context;
 759         int ret;
 760         unsigned long flags;
 761
 762         spin_lock_irqsave(&iommu->lock, flags);
 763         root = &iommu->root_entry[bus];
 764         context = get_context_addr_from_root(root);
 765         if (!context) {
 766                 ret = 0;
 767                 goto out;
 768         }
 769         ret = context_present(&context[devfn]);
 770 out:
 771         spin_unlock_irqrestore(&iommu->lock, flags);
 772         return ret;
 773 }
 774
 775 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 776 {
 777         struct root_entry *root;
 778         struct context_entry *context;
 779         unsigned long flags;
 780
 781         spin_lock_irqsave(&iommu->lock, flags);
 782         root = &iommu->root_entry[bus];
 783         context = get_context_addr_from_root(root);
 784         if (context) {
 785                 context_clear_entry(&context[devfn]);
 786                 __iommu_flush_cache(iommu, &context[devfn], \
 787                         sizeof(*context));
 788         }
 789         spin_unlock_irqrestore(&iommu->lock, flags);
 790 }
 791
 792 static void free_context_table(struct intel_iommu *iommu)
 793 {
 794         struct root_entry *root;
 795         int i;
 796         unsigned long flags;
 797         struct context_entry *context;
 798
 799         spin_lock_irqsave(&iommu->lock, flags);
 800         if (!iommu->root_entry) {
 801                 goto out;
 802         }
 803         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 804                 root = &iommu->root_entry[i];
 805                 context = get_context_addr_from_root(root);
 806                 if (context)
 807                         free_pgtable_page(context);
 808         }
 809         free_pgtable_page(iommu->root_entry);
 810         iommu->root_entry = NULL;
 811 out:
 812         spin_unlock_irqrestore(&iommu->lock, flags);
 813 }
 814
 815 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 816                                       unsigned long pfn, int *target_level)
 817 {
 818         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 819         struct dma_pte *parent, *pte = NULL;
 820         int level = agaw_to_level(domain->agaw);
 821         int offset;
 822
 823         BUG_ON(!domain->pgd);
 824
 825         if (addr_width < BITS_PER_LONG && pfn >> addr_width)
 826                 /* Address beyond IOMMU's addressing capabilities. */
 827                 return NULL;
 828
 829         parent = domain->pgd;
 830
 831         while (1) {
 832                 void *tmp_page;
 833
 834                 offset = pfn_level_offset(pfn, level);
 835                 pte = &parent[offset];
 836                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 837                         break;
 838                 if (level == *target_level)
 839                         break;
 840
 841                 if (!dma_pte_present(pte)) {
 842                         uint64_t pteval;
 843
 844                         tmp_page = alloc_pgtable_page(domain->nid);
 845
 846                         if (!tmp_page)
 847                                 return NULL;
 848
 849                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 850                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 851                         if (cmpxchg64(&pte->val, 0ULL, pteval)) {
 852                                 /* Someone else set it while we were thinking; use theirs. */
 853                                 free_pgtable_page(tmp_page);
 854                         } else {
 855                                 dma_pte_addr(pte);
 856                                 domain_flush_cache(domain, pte, sizeof(*pte));
 857                         }
 858                 }
 859                 if (level == 1)
 860                         break;
 861
 862                 parent = phys_to_virt(dma_pte_addr(pte));
 863                 level--;
 864         }
 865
 866         if (!*target_level)
 867                 *target_level = level;
 868
 869         return pte;
 870 }
 871
 872
 873 /* return address's pte at specific level */
 874 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 875                                          unsigned long pfn,
 876                                          int level, int *large_page)
 877 {
 878         struct dma_pte *parent, *pte = NULL;
 879         int total = agaw_to_level(domain->agaw);
 880         int offset;
 881
 882         parent = domain->pgd;
 883         while (level <= total) {
 884                 offset = pfn_level_offset(pfn, total);
 885                 pte = &parent[offset];
 886                 if (level == total)
 887                         return pte;
 888
 889                 if (!dma_pte_present(pte)) {
 890                         *large_page = total;
 891                         break;
 892                 }
 893
 894                 if (pte->val & DMA_PTE_LARGE_PAGE) {
 895                         *large_page = total;
 896                         return pte;
 897                 }
 898
 899                 parent = phys_to_virt(dma_pte_addr(pte));
 900                 total--;
 901         }
 902         return NULL;
 903 }
 904
 905 /* clear last level pte, a tlb flush should be followed */
 906 static void dma_pte_clear_range(struct dmar_domain *domain,
 907                                 unsigned long start_pfn,
 908                                 unsigned long last_pfn)
 909 {
 910         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 911         unsigned int large_page = 1;
 912         struct dma_pte *first_pte, *pte;
 913
 914         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 915         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 916         BUG_ON(start_pfn > last_pfn);
 917
 918         /* we don't need lock here; nobody else touches the iova range */
 919         do {
 920                 large_page = 1;
 921                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
 922                 if (!pte) {
 923                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
 924                         continue;
 925                 }
 926                 do {
 927                         dma_clear_pte(pte);
 928                         start_pfn += lvl_to_nr_pages(large_page);
 929                         pte++;
 930                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
 931
 932                 domain_flush_cache(domain, first_pte,
 933                                    (void *)pte - (void *)first_pte);
 934
 935         } while (start_pfn && start_pfn <= last_pfn);
 936 }
 937
 938 static void dma_pte_free_level(struct dmar_domain *domain, int level,
 939                                struct dma_pte *pte, unsigned long pfn,
 940                                unsigned long start_pfn, unsigned long last_pfn)
 941 {
 942         pfn = max(start_pfn, pfn);
 943         pte = &pte[pfn_level_offset(pfn, level)];
 944
 945         do {
 946                 unsigned long level_pfn;
 947                 struct dma_pte *level_pte;
 948
 949                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
 950                         goto next;
 951
 952                 level_pfn = pfn & level_mask(level - 1);
 953                 level_pte = phys_to_virt(dma_pte_addr(pte));
 954
 955                 if (level > 2)
 956                         dma_pte_free_level(domain, level - 1, level_pte,
 957                                            level_pfn, start_pfn, last_pfn);
 958
 959                 /* If range covers entire pagetable, free it */
 960                 if (!(start_pfn > level_pfn ||
 961                       last_pfn < level_pfn + level_size(level) - 1)) {
 962                         dma_clear_pte(pte);
 963                         domain_flush_cache(domain, pte, sizeof(*pte));
 964                         free_pgtable_page(level_pte);
 965                 }
 966 next:
 967                 pfn += level_size(level);
 968         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
 969 }
 970
 971 /* free page table pages. last level pte should already be cleared */
 972 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 973                                    unsigned long start_pfn,
 974                                    unsigned long last_pfn)
 975 {
 976         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 977
 978         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 979         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 980         BUG_ON(start_pfn > last_pfn);
 981
 982         /* We don't need lock here; nobody else touches the iova range */
 983         dma_pte_free_level(domain, agaw_to_level(domain->agaw),
 984                            domain->pgd, 0, start_pfn, last_pfn);
 985
 986         /* free pgd */
 987         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
 988                 free_pgtable_page(domain->pgd);
 989                 domain->pgd = NULL;
 990         }
 991 }
 992
 993 /* When a page at a given level is being unlinked from its parent, we don't
 994    need to *modify* it at all. All we need to do is make a list of all the
 995    pages which can be freed just as soon as we've flushed the IOTLB and we
 996    know the hardware page-walk will no longer touch them.
 997    The 'pte' argument is the *parent* PTE, pointing to the page that is to
 998    be freed. */
 999 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1000                                             int level, struct dma_pte *pte,
1001                                             struct page *freelist)
1002 {
1003         struct page *pg;
1004
1005         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1006         pg->freelist = freelist;
1007         freelist = pg;
1008
1009         if (level == 1)
1010                 return freelist;
1011
1012         for (pte = page_address(pg); !first_pte_in_page(pte); pte++) {
1013                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1014                         freelist = dma_pte_list_pagetables(domain, level - 1,
1015                                                            pte, freelist);
1016         }
1017
1018         return freelist;
1019 }
1020
1021 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1022                                         struct dma_pte *pte, unsigned long pfn,
1023                                         unsigned long start_pfn,
1024                                         unsigned long last_pfn,
1025                                         struct page *freelist)
1026 {
1027         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1028
1029         pfn = max(start_pfn, pfn);
1030         pte = &pte[pfn_level_offset(pfn, level)];
1031
1032         do {
1033                 unsigned long level_pfn;
1034
1035                 if (!dma_pte_present(pte))
1036                         goto next;
1037
1038                 level_pfn = pfn & level_mask(level);
1039
1040                 /* If range covers entire pagetable, free it */
1041                 if (start_pfn <= level_pfn &&
1042                     last_pfn >= level_pfn + level_size(level) - 1) {
1043                         /* These suborbinate page tables are going away entirely. Don't
1044                            bother to clear them; we're just going to *free* them. */
1045                         if (level > 1 && !dma_pte_superpage(pte))
1046                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1047
1048                         dma_clear_pte(pte);
1049                         if (!first_pte)
1050                                 first_pte = pte;
1051                         last_pte = pte;
1052                 } else if (level > 1) {
1053                         /* Recurse down into a level that isn't *entirely* obsolete */
1054                         freelist = dma_pte_clear_level(domain, level - 1,
1055                                                        phys_to_virt(dma_pte_addr(pte)),
1056                                                        level_pfn, start_pfn, last_pfn,
1057                                                        freelist);
1058                 }
1059 next:
1060                 pfn += level_size(level);
1061         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1062
1063         if (first_pte)
1064                 domain_flush_cache(domain, first_pte,
1065                                    (void *)++last_pte - (void *)first_pte);
1066
1067         return freelist;
1068 }
1069
1070 /* We can't just free the pages because the IOMMU may still be walking
1071    the page tables, and may have cached the intermediate levels. The
1072    pages can only be freed after the IOTLB flush has been done. */
1073 struct page *domain_unmap(struct dmar_domain *domain,
1074                           unsigned long start_pfn,
1075                           unsigned long last_pfn)
1076 {
1077         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1078         struct page *freelist = NULL;
1079
1080         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
1081         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
1082         BUG_ON(start_pfn > last_pfn);
1083
1084         /* we don't need lock here; nobody else touches the iova range */
1085         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1086                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1087
1088         /* free pgd */
1089         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1090                 struct page *pgd_page = virt_to_page(domain->pgd);
1091                 pgd_page->freelist = freelist;
1092                 freelist = pgd_page;
1093
1094                 domain->pgd = NULL;
1095         }
1096
1097         return freelist;
1098 }
1099
1100 void dma_free_pagelist(struct page *freelist)
1101 {
1102         struct page *pg;
1103
1104         while ((pg = freelist)) {
1105                 freelist = pg->freelist;
1106                 free_pgtable_page(page_address(pg));
1107         }
1108 }
1109
1110 /* iommu handling */
1111 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1112 {
1113         struct root_entry *root;
1114         unsigned long flags;
1115
1116         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1117         if (!root)
1118                 return -ENOMEM;
1119
1120         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1121
1122         spin_lock_irqsave(&iommu->lock, flags);
1123         iommu->root_entry = root;
1124         spin_unlock_irqrestore(&iommu->lock, flags);
1125
1126         return 0;
1127 }
1128
1129 static void iommu_set_root_entry(struct intel_iommu *iommu)
1130 {
1131         void *addr;
1132         u32 sts;
1133         unsigned long flag;
1134
1135         addr = iommu->root_entry;
1136
1137         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1138         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
1139
1140         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1141
1142         /* Make sure hardware complete it */
1143         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1144                       readl, (sts & DMA_GSTS_RTPS), sts);
1145
1146         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1147 }
1148
1149 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1150 {
1151         u32 val;
1152         unsigned long flag;
1153
1154         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1155                 return;
1156
1157         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1158         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1159
1160         /* Make sure hardware complete it */
1161         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1162                       readl, (!(val & DMA_GSTS_WBFS)), val);
1163
1164         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1165 }
1166
1167 /* return value determine if we need a write buffer flush */
1168 static void __iommu_flush_context(struct intel_iommu *iommu,
1169                                   u16 did, u16 source_id, u8 function_mask,
1170                                   u64 type)
1171 {
1172         u64 val = 0;
1173         unsigned long flag;
1174
1175         switch (type) {
1176         case DMA_CCMD_GLOBAL_INVL:
1177                 val = DMA_CCMD_GLOBAL_INVL;
1178                 break;
1179         case DMA_CCMD_DOMAIN_INVL:
1180                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1181                 break;
1182         case DMA_CCMD_DEVICE_INVL:
1183                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1184                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1185                 break;
1186         default:
1187                 BUG();
1188         }
1189         val |= DMA_CCMD_ICC;
1190
1191         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1192         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1193
1194         /* Make sure hardware complete it */
1195         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1196                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1197
1198         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1199 }
1200
1201 /* return value determine if we need a write buffer flush */
1202 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1203                                 u64 addr, unsigned int size_order, u64 type)
1204 {
1205         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1206         u64 val = 0, val_iva = 0;
1207         unsigned long flag;
1208
1209         switch (type) {
1210         case DMA_TLB_GLOBAL_FLUSH:
1211                 /* global flush doesn't need set IVA_REG */
1212                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1213                 break;
1214         case DMA_TLB_DSI_FLUSH:
1215                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1216                 break;
1217         case DMA_TLB_PSI_FLUSH:
1218                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1219                 /* IH bit is passed in as part of address */
1220                 val_iva = size_order | addr;
1221                 break;
1222         default:
1223                 BUG();
1224         }
1225         /* Note: set drain read/write */
1226 #if 0
1227         /*
1228          * This is probably to be super secure.. Looks like we can
1229          * ignore it without any impact.
1230          */
1231         if (cap_read_drain(iommu->cap))
1232                 val |= DMA_TLB_READ_DRAIN;
1233 #endif
1234         if (cap_write_drain(iommu->cap))
1235                 val |= DMA_TLB_WRITE_DRAIN;
1236
1237         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1238         /* Note: Only uses first TLB reg currently */
1239         if (val_iva)
1240                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1241         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1242
1243         /* Make sure hardware complete it */
1244         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1245                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1246
1247         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1248
1249         /* check IOTLB invalidation granularity */
1250         if (DMA_TLB_IAIG(val) == 0)
1251                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1252         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1253                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1254                         (unsigned long long)DMA_TLB_IIRG(type),
1255                         (unsigned long long)DMA_TLB_IAIG(val));
1256 }
1257
1258 static struct device_domain_info *
1259 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1260                          u8 bus, u8 devfn)
1261 {
1262         int found = 0;
1263         unsigned long flags;
1264         struct device_domain_info *info;
1265         struct pci_dev *pdev;
1266
1267         if (!ecap_dev_iotlb_support(iommu->ecap))
1268                 return NULL;
1269
1270         if (!iommu->qi)
1271                 return NULL;
1272
1273         spin_lock_irqsave(&device_domain_lock, flags);
1274         list_for_each_entry(info, &domain->devices, link)
1275                 if (info->bus == bus && info->devfn == devfn) {
1276                         found = 1;
1277                         break;
1278                 }
1279         spin_unlock_irqrestore(&device_domain_lock, flags);
1280
1281         if (!found || !info->dev || !dev_is_pci(info->dev))
1282                 return NULL;
1283
1284         pdev = to_pci_dev(info->dev);
1285
1286         if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
1287                 return NULL;
1288
1289         if (!dmar_find_matched_atsr_unit(pdev))
1290                 return NULL;
1291
1292         return info;
1293 }
1294
1295 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1296 {
1297         if (!info || !dev_is_pci(info->dev))
1298                 return;
1299
1300         pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
1301 }
1302
1303 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1304 {
1305         if (!info->dev || !dev_is_pci(info->dev) ||
1306             !pci_ats_enabled(to_pci_dev(info->dev)))
1307                 return;
1308
1309         pci_disable_ats(to_pci_dev(info->dev));
1310 }
1311
1312 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1313                                   u64 addr, unsigned mask)
1314 {
1315         u16 sid, qdep;
1316         unsigned long flags;
1317         struct device_domain_info *info;
1318
1319         spin_lock_irqsave(&device_domain_lock, flags);
1320         list_for_each_entry(info, &domain->devices, link) {
1321                 struct pci_dev *pdev;
1322                 if (!info->dev || !dev_is_pci(info->dev))
1323                         continue;
1324
1325                 pdev = to_pci_dev(info->dev);
1326                 if (!pci_ats_enabled(pdev))
1327                         continue;
1328
1329                 sid = info->bus << 8 | info->devfn;
1330                 qdep = pci_ats_queue_depth(pdev);
1331                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1332         }
1333         spin_unlock_irqrestore(&device_domain_lock, flags);
1334 }
1335
1336 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1337                                   unsigned long pfn, unsigned int pages, int ih, int map)
1338 {
1339         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1340         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1341
1342         BUG_ON(pages == 0);
1343
1344         if (ih)
1345                 ih = 1 << 6;
1346         /*
1347          * Fallback to domain selective flush if no PSI support or the size is
1348          * too big.
1349          * PSI requires page size to be 2 ^ x, and the base address is naturally
1350          * aligned to the size
1351          */
1352         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1353                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1354                                                 DMA_TLB_DSI_FLUSH);
1355         else
1356                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1357                                                 DMA_TLB_PSI_FLUSH);
1358
1359         /*
1360          * In caching mode, changes of pages from non-present to present require
1361          * flush. However, device IOTLB doesn't need to be flushed in this case.
1362          */
1363         if (!cap_caching_mode(iommu->cap) || !map)
1364                 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1365 }
1366
1367 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1368 {
1369         u32 pmen;
1370         unsigned long flags;
1371
1372         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1373         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1374         pmen &= ~DMA_PMEN_EPM;
1375         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1376
1377         /* wait for the protected region status bit to clear */
1378         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1379                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1380
1381         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1382 }
1383
1384 static int iommu_enable_translation(struct intel_iommu *iommu)
1385 {
1386         u32 sts;
1387         unsigned long flags;
1388
1389         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1390         iommu->gcmd |= DMA_GCMD_TE;
1391         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1392
1393         /* Make sure hardware complete it */
1394         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1395                       readl, (sts & DMA_GSTS_TES), sts);
1396
1397         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1398         return 0;
1399 }
1400
1401 static int iommu_disable_translation(struct intel_iommu *iommu)
1402 {
1403         u32 sts;
1404         unsigned long flag;
1405
1406         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1407         iommu->gcmd &= ~DMA_GCMD_TE;
1408         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1409
1410         /* Make sure hardware complete it */
1411         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1412                       readl, (!(sts & DMA_GSTS_TES)), sts);
1413
1414         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1415         return 0;
1416 }
1417
1418
1419 static int iommu_init_domains(struct intel_iommu *iommu)
1420 {
1421         unsigned long ndomains;
1422         unsigned long nlongs;
1423
1424         ndomains = cap_ndoms(iommu->cap);
1425         pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1426                  iommu->seq_id, ndomains);
1427         nlongs = BITS_TO_LONGS(ndomains);
1428
1429         spin_lock_init(&iommu->lock);
1430
1431         /* TBD: there might be 64K domains,
1432          * consider other allocation for future chip
1433          */
1434         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1435         if (!iommu->domain_ids) {
1436                 pr_err("IOMMU%d: allocating domain id array failed\n",
1437                        iommu->seq_id);
1438                 return -ENOMEM;
1439         }
1440         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1441                         GFP_KERNEL);
1442         if (!iommu->domains) {
1443                 pr_err("IOMMU%d: allocating domain array failed\n",
1444                        iommu->seq_id);
1445                 kfree(iommu->domain_ids);
1446                 iommu->domain_ids = NULL;
1447                 return -ENOMEM;
1448         }
1449
1450         /*
1451          * if Caching mode is set, then invalid translations are tagged
1452          * with domainid 0. Hence we need to pre-allocate it.
1453          */
1454         if (cap_caching_mode(iommu->cap))
1455                 set_bit(0, iommu->domain_ids);
1456         return 0;
1457 }
1458
1459 static void free_dmar_iommu(struct intel_iommu *iommu)
1460 {
1461         struct dmar_domain *domain;
1462         int i, count;
1463         unsigned long flags;
1464
1465         if ((iommu->domains) && (iommu->domain_ids)) {
1466                 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1467                         /*
1468                          * Domain id 0 is reserved for invalid translation
1469                          * if hardware supports caching mode.
1470                          */
1471                         if (cap_caching_mode(iommu->cap) && i == 0)
1472                                 continue;
1473
1474                         domain = iommu->domains[i];
1475                         clear_bit(i, iommu->domain_ids);
1476
1477                         spin_lock_irqsave(&domain->iommu_lock, flags);
1478                         count = --domain->iommu_count;
1479                         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1480                         if (count == 0)
1481                                 domain_exit(domain);
1482                 }
1483         }
1484
1485         if (iommu->gcmd & DMA_GCMD_TE)
1486                 iommu_disable_translation(iommu);
1487
1488         kfree(iommu->domains);
1489         kfree(iommu->domain_ids);
1490         iommu->domains = NULL;
1491         iommu->domain_ids = NULL;
1492
1493         g_iommus[iommu->seq_id] = NULL;
1494
1495         /* free context mapping */
1496         free_context_table(iommu);
1497 }
1498
1499 static struct dmar_domain *alloc_domain(bool vm)
1500 {
1501         /* domain id for virtual machine, it won't be set in context */
1502         static atomic_t vm_domid = ATOMIC_INIT(0);
1503         struct dmar_domain *domain;
1504
1505         domain = alloc_domain_mem();
1506         if (!domain)
1507                 return NULL;
1508
1509         domain->nid = -1;
1510         domain->iommu_count = 0;
1511         memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
1512         domain->flags = 0;
1513         spin_lock_init(&domain->iommu_lock);
1514         INIT_LIST_HEAD(&domain->devices);
1515         if (vm) {
1516                 domain->id = atomic_inc_return(&vm_domid);
1517                 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
1518         }
1519
1520         return domain;
1521 }
1522
1523 static int iommu_attach_domain(struct dmar_domain *domain,
1524                                struct intel_iommu *iommu)
1525 {
1526         int num;
1527         unsigned long ndomains;
1528         unsigned long flags;
1529
1530         ndomains = cap_ndoms(iommu->cap);
1531
1532         spin_lock_irqsave(&iommu->lock, flags);
1533
1534         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1535         if (num >= ndomains) {
1536                 spin_unlock_irqrestore(&iommu->lock, flags);
1537                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1538                 return -ENOMEM;
1539         }
1540
1541         domain->id = num;
1542         domain->iommu_count++;
1543         set_bit(num, iommu->domain_ids);
1544         set_bit(iommu->seq_id, domain->iommu_bmp);
1545         iommu->domains[num] = domain;
1546         spin_unlock_irqrestore(&iommu->lock, flags);
1547
1548         return 0;
1549 }
1550
1551 static void iommu_detach_domain(struct dmar_domain *domain,
1552                                 struct intel_iommu *iommu)
1553 {
1554         unsigned long flags;
1555         int num, ndomains;
1556
1557         spin_lock_irqsave(&iommu->lock, flags);
1558         ndomains = cap_ndoms(iommu->cap);
1559         for_each_set_bit(num, iommu->domain_ids, ndomains) {
1560                 if (iommu->domains[num] == domain) {
1561                         clear_bit(num, iommu->domain_ids);
1562                         iommu->domains[num] = NULL;
1563                         break;
1564                 }
1565         }
1566         spin_unlock_irqrestore(&iommu->lock, flags);
1567 }
1568
1569 static struct iova_domain reserved_iova_list;
1570 static struct lock_class_key reserved_rbtree_key;
1571
1572 static int dmar_init_reserved_ranges(void)
1573 {
1574         struct pci_dev *pdev = NULL;
1575         struct iova *iova;
1576         int i;
1577
1578         init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1579
1580         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1581                 &reserved_rbtree_key);
1582
1583         /* IOAPIC ranges shouldn't be accessed by DMA */
1584         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1585                 IOVA_PFN(IOAPIC_RANGE_END));
1586         if (!iova) {
1587                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1588                 return -ENODEV;
1589         }
1590
1591         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1592         for_each_pci_dev(pdev) {
1593                 struct resource *r;
1594
1595                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1596                         r = &pdev->resource[i];
1597                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1598                                 continue;
1599                         iova = reserve_iova(&reserved_iova_list,
1600                                             IOVA_PFN(r->start),
1601                                             IOVA_PFN(r->end));
1602                         if (!iova) {
1603                                 printk(KERN_ERR "Reserve iova failed\n");
1604                                 return -ENODEV;
1605                         }
1606                 }
1607         }
1608         return 0;
1609 }
1610
1611 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1612 {
1613         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1614 }
1615
1616 static inline int guestwidth_to_adjustwidth(int gaw)
1617 {
1618         int agaw;
1619         int r = (gaw - 12) % 9;
1620
1621         if (r == 0)
1622                 agaw = gaw;
1623         else
1624                 agaw = gaw + 9 - r;
1625         if (agaw > 64)
1626                 agaw = 64;
1627         return agaw;
1628 }
1629
1630 static int domain_init(struct dmar_domain *domain, int guest_width)
1631 {
1632         struct intel_iommu *iommu;
1633         int adjust_width, agaw;
1634         unsigned long sagaw;
1635
1636         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1637         domain_reserve_special_ranges(domain);
1638
1639         /* calculate AGAW */
1640         iommu = domain_get_iommu(domain);
1641         if (guest_width > cap_mgaw(iommu->cap))
1642                 guest_width = cap_mgaw(iommu->cap);
1643         domain->gaw = guest_width;
1644         adjust_width = guestwidth_to_adjustwidth(guest_width);
1645         agaw = width_to_agaw(adjust_width);
1646         sagaw = cap_sagaw(iommu->cap);
1647         if (!test_bit(agaw, &sagaw)) {
1648                 /* hardware doesn't support it, choose a bigger one */
1649                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1650                 agaw = find_next_bit(&sagaw, 5, agaw);
1651                 if (agaw >= 5)
1652                         return -ENODEV;
1653         }
1654         domain->agaw = agaw;
1655
1656         if (ecap_coherent(iommu->ecap))
1657                 domain->iommu_coherency = 1;
1658         else
1659                 domain->iommu_coherency = 0;
1660
1661         if (ecap_sc_support(iommu->ecap))
1662                 domain->iommu_snooping = 1;
1663         else
1664                 domain->iommu_snooping = 0;
1665
1666         if (intel_iommu_superpage)
1667                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1668         else
1669                 domain->iommu_superpage = 0;
1670
1671         domain->nid = iommu->node;
1672
1673         /* always allocate the top pgd */
1674         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1675         if (!domain->pgd)
1676                 return -ENOMEM;
1677         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1678         return 0;
1679 }
1680
1681 static void domain_exit(struct dmar_domain *domain)
1682 {
1683         struct dmar_drhd_unit *drhd;
1684         struct intel_iommu *iommu;
1685         struct page *freelist = NULL;
1686
1687         /* Domain 0 is reserved, so dont process it */
1688         if (!domain)
1689                 return;
1690
1691         /* Flush any lazy unmaps that may reference this domain */
1692         if (!intel_iommu_strict)
1693                 flush_unmaps_timeout(0);
1694
1695         /* remove associated devices */
1696         domain_remove_dev_info(domain);
1697
1698         /* destroy iovas */
1699         put_iova_domain(&domain->iovad);
1700
1701         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1702
1703         /* clear attached or cached domains */
1704         rcu_read_lock();
1705         for_each_active_iommu(iommu, drhd)
1706                 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1707                     test_bit(iommu->seq_id, domain->iommu_bmp))
1708                         iommu_detach_domain(domain, iommu);
1709         rcu_read_unlock();
1710
1711         dma_free_pagelist(freelist);
1712
1713         free_domain_mem(domain);
1714 }
1715
1716 static int domain_context_mapping_one(struct dmar_domain *domain,
1717                                       struct intel_iommu *iommu,
1718                                       u8 bus, u8 devfn, int translation)
1719 {
1720         struct context_entry *context;
1721         unsigned long flags;
1722         struct dma_pte *pgd;
1723         unsigned long num;
1724         unsigned long ndomains;
1725         int id;
1726         int agaw;
1727         struct device_domain_info *info = NULL;
1728
1729         pr_debug("Set context mapping for %02x:%02x.%d\n",
1730                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1731
1732         BUG_ON(!domain->pgd);
1733         BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1734                translation != CONTEXT_TT_MULTI_LEVEL);
1735
1736         context = device_to_context_entry(iommu, bus, devfn);
1737         if (!context)
1738                 return -ENOMEM;
1739         spin_lock_irqsave(&iommu->lock, flags);
1740         if (context_present(context)) {
1741                 spin_unlock_irqrestore(&iommu->lock, flags);
1742                 return 0;
1743         }
1744
1745         id = domain->id;
1746         pgd = domain->pgd;
1747
1748         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1749             domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1750                 int found = 0;
1751
1752                 /* find an available domain id for this device in iommu */
1753                 ndomains = cap_ndoms(iommu->cap);
1754                 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1755                         if (iommu->domains[num] == domain) {
1756                                 id = num;
1757                                 found = 1;
1758                                 break;
1759                         }
1760                 }
1761
1762                 if (found == 0) {
1763                         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1764                         if (num >= ndomains) {
1765                                 spin_unlock_irqrestore(&iommu->lock, flags);
1766                                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1767                                 return -EFAULT;
1768                         }
1769
1770                         set_bit(num, iommu->domain_ids);
1771                         iommu->domains[num] = domain;
1772                         id = num;
1773                 }
1774
1775                 /* Skip top levels of page tables for
1776                  * iommu which has less agaw than default.
1777                  * Unnecessary for PT mode.
1778                  */
1779                 if (translation != CONTEXT_TT_PASS_THROUGH) {
1780                         for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1781                                 pgd = phys_to_virt(dma_pte_addr(pgd));
1782                                 if (!dma_pte_present(pgd)) {
1783                                         spin_unlock_irqrestore(&iommu->lock, flags);
1784                                         return -ENOMEM;
1785                                 }
1786                         }
1787                 }
1788         }
1789
1790         context_set_domain_id(context, id);
1791
1792         if (translation != CONTEXT_TT_PASS_THROUGH) {
1793                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
1794                 translation = info ? CONTEXT_TT_DEV_IOTLB :
1795                                      CONTEXT_TT_MULTI_LEVEL;
1796         }
1797         /*
1798          * In pass through mode, AW must be programmed to indicate the largest
1799          * AGAW value supported by hardware. And ASR is ignored by hardware.
1800          */
1801         if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1802                 context_set_address_width(context, iommu->msagaw);
1803         else {
1804                 context_set_address_root(context, virt_to_phys(pgd));
1805                 context_set_address_width(context, iommu->agaw);
1806         }
1807
1808         context_set_translation_type(context, translation);
1809         context_set_fault_enable(context);
1810         context_set_present(context);
1811         domain_flush_cache(domain, context, sizeof(*context));
1812
1813         /*
1814          * It's a non-present to present mapping. If hardware doesn't cache
1815          * non-present entry we only need to flush the write-buffer. If the
1816          * _does_ cache non-present entries, then it does so in the special
1817          * domain #0, which we have to flush:
1818          */
1819         if (cap_caching_mode(iommu->cap)) {
1820                 iommu->flush.flush_context(iommu, 0,
1821                                            (((u16)bus) << 8) | devfn,
1822                                            DMA_CCMD_MASK_NOBIT,
1823                                            DMA_CCMD_DEVICE_INVL);
1824                 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1825         } else {
1826                 iommu_flush_write_buffer(iommu);
1827         }
1828         iommu_enable_dev_iotlb(info);
1829         spin_unlock_irqrestore(&iommu->lock, flags);
1830
1831         spin_lock_irqsave(&domain->iommu_lock, flags);
1832         if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1833                 domain->iommu_count++;
1834                 if (domain->iommu_count == 1)
1835                         domain->nid = iommu->node;
1836                 domain_update_iommu_cap(domain);
1837         }
1838         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1839         return 0;
1840 }
1841
1842 static int
1843 domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1844                        int translation)
1845 {
1846         int ret;
1847         struct pci_dev *pdev, *tmp, *parent;
1848         struct intel_iommu *iommu;
1849         u8 bus, devfn;
1850
1851         iommu = device_to_iommu(dev, &bus, &devfn);
1852         if (!iommu)
1853                 return -ENODEV;
1854
1855         ret = domain_context_mapping_one(domain, iommu, bus, devfn,
1856                                          translation);
1857         if (ret || !dev_is_pci(dev))
1858                 return ret;
1859
1860         /* dependent device mapping */
1861         pdev = to_pci_dev(dev);
1862         tmp = pci_find_upstream_pcie_bridge(pdev);
1863         if (!tmp)
1864                 return 0;
1865         /* Secondary interface's bus number and devfn 0 */
1866         parent = pdev->bus->self;
1867         while (parent != tmp) {
1868                 ret = domain_context_mapping_one(domain, iommu,
1869                                                  parent->bus->number,
1870                                                  parent->devfn, translation);
1871                 if (ret)
1872                         return ret;
1873                 parent = parent->bus->self;
1874         }
1875         if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1876                 return domain_context_mapping_one(domain, iommu,
1877                                         tmp->subordinate->number, 0,
1878                                         translation);
1879         else /* this is a legacy PCI bridge */
1880                 return domain_context_mapping_one(domain, iommu,
1881                                                   tmp->bus->number,
1882                                                   tmp->devfn,
1883                                                   translation);
1884 }
1885
1886 static int domain_context_mapped(struct device *dev)
1887 {
1888         int ret;
1889         struct pci_dev *pdev, *tmp, *parent;
1890         struct intel_iommu *iommu;
1891         u8 bus, devfn;
1892
1893         iommu = device_to_iommu(dev, &bus, &devfn);
1894         if (!iommu)
1895                 return -ENODEV;
1896
1897         ret = device_context_mapped(iommu, bus, devfn);
1898         if (!ret || !dev_is_pci(dev))
1899                 return ret;
1900
1901         /* dependent device mapping */
1902         pdev = to_pci_dev(dev);
1903         tmp = pci_find_upstream_pcie_bridge(pdev);
1904         if (!tmp)
1905                 return ret;
1906         /* Secondary interface's bus number and devfn 0 */
1907         parent = pdev->bus->self;
1908         while (parent != tmp) {
1909                 ret = device_context_mapped(iommu, parent->bus->number,
1910                                             parent->devfn);
1911                 if (!ret)
1912                         return ret;
1913                 parent = parent->bus->self;
1914         }
1915         if (pci_is_pcie(tmp))
1916                 return device_context_mapped(iommu, tmp->subordinate->number,
1917                                              0);
1918         else
1919                 return device_context_mapped(iommu, tmp->bus->number,
1920                                              tmp->devfn);
1921 }
1922
1923 /* Returns a number of VTD pages, but aligned to MM page size */
1924 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1925                                             size_t size)
1926 {
1927         host_addr &= ~PAGE_MASK;
1928         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1929 }
1930
1931 /* Return largest possible superpage level for a given mapping */
1932 static inline int hardware_largepage_caps(struct dmar_domain *domain,
1933                                           unsigned long iov_pfn,
1934                                           unsigned long phy_pfn,
1935                                           unsigned long pages)
1936 {
1937         int support, level = 1;
1938         unsigned long pfnmerge;
1939
1940         support = domain->iommu_superpage;
1941
1942         /* To use a large page, the virtual *and* physical addresses
1943            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1944            of them will mean we have to use smaller pages. So just
1945            merge them and check both at once. */
1946         pfnmerge = iov_pfn | phy_pfn;
1947
1948         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1949                 pages >>= VTD_STRIDE_SHIFT;
1950                 if (!pages)
1951                         break;
1952                 pfnmerge >>= VTD_STRIDE_SHIFT;
1953                 level++;
1954                 support--;
1955         }
1956         return level;
1957 }
1958
1959 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1960                             struct scatterlist *sg, unsigned long phys_pfn,
1961                             unsigned long nr_pages, int prot)
1962 {
1963         struct dma_pte *first_pte = NULL, *pte = NULL;
1964         phys_addr_t uninitialized_var(pteval);
1965         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1966         unsigned long sg_res;
1967         unsigned int largepage_lvl = 0;
1968         unsigned long lvl_pages = 0;
1969
1970         BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1971
1972         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1973                 return -EINVAL;
1974
1975         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1976
1977         if (sg)
1978                 sg_res = 0;
1979         else {
1980                 sg_res = nr_pages + 1;
1981                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1982         }
1983
1984         while (nr_pages > 0) {
1985                 uint64_t tmp;
1986
1987                 if (!sg_res) {
1988                         sg_res = aligned_nrpages(sg->offset, sg->length);
1989                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1990                         sg->dma_length = sg->length;
1991                         pteval = page_to_phys(sg_page(sg)) | prot;
1992                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
1993                 }
1994
1995                 if (!pte) {
1996                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1997
1998                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
1999                         if (!pte)
2000                                 return -ENOMEM;
2001                         /* It is large page*/
2002                         if (largepage_lvl > 1) {
2003                                 pteval |= DMA_PTE_LARGE_PAGE;
2004                                 /* Ensure that old small page tables are removed to make room
2005                                    for superpage, if they exist. */
2006                                 dma_pte_clear_range(domain, iov_pfn,
2007                                                     iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
2008                                 dma_pte_free_pagetable(domain, iov_pfn,
2009                                                        iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
2010                         } else {
2011                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2012                         }
2013
2014                 }
2015                 /* We don't need lock here, nobody else
2016                  * touches the iova range
2017                  */
2018                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2019                 if (tmp) {
2020                         static int dumps = 5;
2021                         printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2022                                iov_pfn, tmp, (unsigned long long)pteval);
2023                         if (dumps) {
2024                                 dumps--;
2025                                 debug_dma_dump_mappings(NULL);
2026                         }
2027                         WARN_ON(1);
2028                 }
2029
2030                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2031
2032                 BUG_ON(nr_pages < lvl_pages);
2033                 BUG_ON(sg_res < lvl_pages);
2034
2035                 nr_pages -= lvl_pages;
2036                 iov_pfn += lvl_pages;
2037                 phys_pfn += lvl_pages;
2038                 pteval += lvl_pages * VTD_PAGE_SIZE;
2039                 sg_res -= lvl_pages;
2040
2041                 /* If the next PTE would be the first in a new page, then we
2042                    need to flush the cache on the entries we've just written.
2043                    And then we'll need to recalculate 'pte', so clear it and
2044                    let it get set again in the if (!pte) block above.
2045
2046                    If we're done (!nr_pages) we need to flush the cache too.
2047
2048                    Also if we've been setting superpages, we may need to
2049                    recalculate 'pte' and switch back to smaller pages for the
2050                    end of the mapping, if the trailing size is not enough to
2051                    use another superpage (i.e. sg_res < lvl_pages). */
2052                 pte++;
2053                 if (!nr_pages || first_pte_in_page(pte) ||
2054                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2055                         domain_flush_cache(domain, first_pte,
2056                                            (void *)pte - (void *)first_pte);
2057                         pte = NULL;
2058                 }
2059
2060                 if (!sg_res && nr_pages)
2061                         sg = sg_next(sg);
2062         }
2063         return 0;
2064 }
2065
2066 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2067                                     struct scatterlist *sg, unsigned long nr_pages,
2068                                     int prot)
2069 {
2070         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2071 }
2072
2073 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2074                                      unsigned long phys_pfn, unsigned long nr_pages,
2075                                      int prot)
2076 {
2077         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2078 }
2079
2080 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
2081 {
2082         if (!iommu)
2083                 return;
2084
2085         clear_context_table(iommu, bus, devfn);
2086         iommu->flush.flush_context(iommu, 0, 0, 0,
2087                                            DMA_CCMD_GLOBAL_INVL);
2088         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2089 }
2090
2091 static inline void unlink_domain_info(struct device_domain_info *info)
2092 {
2093         assert_spin_locked(&device_domain_lock);
2094         list_del(&info->link);
2095         list_del(&info->global);
2096         if (info->dev)
2097                 info->dev->archdata.iommu = NULL;
2098 }
2099
2100 static void domain_remove_dev_info(struct dmar_domain *domain)
2101 {
2102         struct device_domain_info *info;
2103         unsigned long flags, flags2;
2104
2105         spin_lock_irqsave(&device_domain_lock, flags);
2106         while (!list_empty(&domain->devices)) {
2107                 info = list_entry(domain->devices.next,
2108                         struct device_domain_info, link);
2109                 unlink_domain_info(info);
2110                 spin_unlock_irqrestore(&device_domain_lock, flags);
2111
2112                 iommu_disable_dev_iotlb(info);
2113                 iommu_detach_dev(info->iommu, info->bus, info->devfn);
2114
2115                 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
2116                         iommu_detach_dependent_devices(info->iommu, info->dev);
2117                         /* clear this iommu in iommu_bmp, update iommu count
2118                          * and capabilities
2119                          */
2120                         spin_lock_irqsave(&domain->iommu_lock, flags2);
2121                         if (test_and_clear_bit(info->iommu->seq_id,
2122                                                domain->iommu_bmp)) {
2123                                 domain->iommu_count--;
2124                                 domain_update_iommu_cap(domain);
2125                         }
2126                         spin_unlock_irqrestore(&domain->iommu_lock, flags2);
2127                 }
2128
2129                 free_devinfo_mem(info);
2130                 spin_lock_irqsave(&device_domain_lock, flags);
2131         }
2132         spin_unlock_irqrestore(&device_domain_lock, flags);
2133 }
2134
2135 /*
2136  * find_domain
2137  * Note: we use struct device->archdata.iommu stores the info
2138  */
2139 static struct dmar_domain *find_domain(struct device *dev)
2140 {
2141         struct device_domain_info *info;
2142
2143         /* No lock here, assumes no domain exit in normal case */
2144         info = dev->archdata.iommu;
2145         if (info)
2146                 return info->domain;
2147         return NULL;
2148 }
2149
2150 static inline struct device_domain_info *
2151 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2152 {
2153         struct device_domain_info *info;
2154
2155         list_for_each_entry(info, &device_domain_list, global)
2156                 if (info->iommu->segment == segment && info->bus == bus &&
2157                     info->devfn == devfn)
2158                         return info;
2159
2160         return NULL;
2161 }
2162
2163 static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
2164                                                 int bus, int devfn,
2165                                                 struct device *dev,
2166                                                 struct dmar_domain *domain)
2167 {
2168         struct dmar_domain *found = NULL;
2169         struct device_domain_info *info;
2170         unsigned long flags;
2171
2172         info = alloc_devinfo_mem();
2173         if (!info)
2174                 return NULL;
2175
2176         info->bus = bus;
2177         info->devfn = devfn;
2178         info->dev = dev;
2179         info->domain = domain;
2180         info->iommu = iommu;
2181         if (!dev)
2182                 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
2183
2184         spin_lock_irqsave(&device_domain_lock, flags);
2185         if (dev)
2186                 found = find_domain(dev);
2187         else {
2188                 struct device_domain_info *info2;
2189                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2190                 if (info2)
2191                         found = info2->domain;
2192         }
2193         if (found) {
2194                 spin_unlock_irqrestore(&device_domain_lock, flags);
2195                 free_devinfo_mem(info);
2196                 /* Caller must free the original domain */
2197                 return found;
2198         }
2199
2200         list_add(&info->link, &domain->devices);
2201         list_add(&info->global, &device_domain_list);
2202         if (dev)
2203                 dev->archdata.iommu = info;
2204         spin_unlock_irqrestore(&device_domain_lock, flags);
2205
2206         return domain;
2207 }
2208
2209 /* domain is initialized */
2210 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2211 {
2212         struct dmar_domain *domain, *free = NULL;
2213         struct intel_iommu *iommu = NULL;
2214         struct device_domain_info *info;
2215         struct pci_dev *dev_tmp = NULL;
2216         unsigned long flags;
2217         u8 bus, devfn, bridge_bus, bridge_devfn;
2218
2219         domain = find_domain(dev);
2220         if (domain)
2221                 return domain;
2222
2223         if (dev_is_pci(dev)) {
2224                 struct pci_dev *pdev = to_pci_dev(dev);
2225                 u16 segment;
2226
2227                 segment = pci_domain_nr(pdev->bus);
2228                 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
2229                 if (dev_tmp) {
2230                         if (pci_is_pcie(dev_tmp)) {
2231                                 bridge_bus = dev_tmp->subordinate->number;
2232                                 bridge_devfn = 0;
2233                         } else {
2234                                 bridge_bus = dev_tmp->bus->number;
2235                                 bridge_devfn = dev_tmp->devfn;
2236                         }
2237                         spin_lock_irqsave(&device_domain_lock, flags);
2238                         info = dmar_search_domain_by_dev_info(segment, bus, devfn);
2239                         if (info) {
2240                                 iommu = info->iommu;
2241                                 domain = info->domain;
2242                         }
2243                         spin_unlock_irqrestore(&device_domain_lock, flags);
2244                         /* pcie-pci bridge already has a domain, uses it */
2245                         if (info)
2246                                 goto found_domain;
2247                 }
2248         }
2249
2250         iommu = device_to_iommu(dev, &bus, &devfn);
2251         if (!iommu)
2252                 goto error;
2253
2254         /* Allocate and initialize new domain for the device */
2255         domain = alloc_domain(false);
2256         if (!domain)
2257                 goto error;
2258         if (iommu_attach_domain(domain, iommu)) {
2259                 free_domain_mem(domain);
2260                 domain = NULL;
2261                 goto error;
2262         }
2263         free = domain;
2264         if (domain_init(domain, gaw))
2265                 goto error;
2266
2267         /* register pcie-to-pci device */
2268         if (dev_tmp) {
2269                 domain = dmar_insert_dev_info(iommu, bridge_bus, bridge_devfn,
2270                                               NULL, domain);
2271                 if (!domain)
2272                         goto error;
2273         }
2274
2275 found_domain:
2276         domain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2277 error:
2278         if (free != domain)
2279                 domain_exit(free);
2280
2281         return domain;
2282 }
2283
2284 static int iommu_identity_mapping;
2285 #define IDENTMAP_ALL            1
2286 #define IDENTMAP_GFX            2
2287 #define IDENTMAP_AZALIA         4
2288
2289 static int iommu_domain_identity_map(struct dmar_domain *domain,
2290                                      unsigned long long start,
2291                                      unsigned long long end)
2292 {
2293         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2294         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2295
2296         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2297                           dma_to_mm_pfn(last_vpfn))) {
2298                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
2299                 return -ENOMEM;
2300         }
2301
2302         pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2303                  start, end, domain->id);
2304         /*
2305          * RMRR range might have overlap with physical memory range,
2306          * clear it first
2307          */
2308         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2309
2310         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2311                                   last_vpfn - first_vpfn + 1,
2312                                   DMA_PTE_READ|DMA_PTE_WRITE);
2313 }
2314
2315 static int iommu_prepare_identity_map(struct device *dev,
2316                                       unsigned long long start,
2317                                       unsigned long long end)
2318 {
2319         struct dmar_domain *domain;
2320         int ret;
2321
2322         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2323         if (!domain)
2324                 return -ENOMEM;
2325
2326         /* For _hardware_ passthrough, don't bother. But for software
2327            passthrough, we do it anyway -- it may indicate a memory
2328            range which is reserved in E820, so which didn't get set
2329            up to start with in si_domain */
2330         if (domain == si_domain && hw_pass_through) {
2331                 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2332                        dev_name(dev), start, end);
2333                 return 0;
2334         }
2335
2336         printk(KERN_INFO
2337                "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2338                dev_name(dev), start, end);
2339
2340         if (end < start) {
2341                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2342                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2343                         dmi_get_system_info(DMI_BIOS_VENDOR),
2344                         dmi_get_system_info(DMI_BIOS_VERSION),
2345                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2346                 ret = -EIO;
2347                 goto error;
2348         }
2349
2350         if (end >> agaw_to_width(domain->agaw)) {
2351                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2352                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2353                      agaw_to_width(domain->agaw),
2354                      dmi_get_system_info(DMI_BIOS_VENDOR),
2355                      dmi_get_system_info(DMI_BIOS_VERSION),
2356                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2357                 ret = -EIO;
2358                 goto error;
2359         }
2360
2361         ret = iommu_domain_identity_map(domain, start, end);
2362         if (ret)
2363                 goto error;
2364
2365         /* context entry init */
2366         ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2367         if (ret)
2368                 goto error;
2369
2370         return 0;
2371
2372  error:
2373         domain_exit(domain);
2374         return ret;
2375 }
2376
2377 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2378                                          struct device *dev)
2379 {
2380         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2381                 return 0;
2382         return iommu_prepare_identity_map(dev, rmrr->base_address,
2383                                           rmrr->end_address);
2384 }
2385
2386 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2387 static inline void iommu_prepare_isa(void)
2388 {
2389         struct pci_dev *pdev;
2390         int ret;
2391
2392         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2393         if (!pdev)
2394                 return;
2395
2396         printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2397         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2398
2399         if (ret)
2400                 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2401                        "floppy might not work\n");
2402
2403 }
2404 #else
2405 static inline void iommu_prepare_isa(void)
2406 {
2407         return;
2408 }
2409 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2410
2411 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2412
2413 static int __init si_domain_init(int hw)
2414 {
2415         struct dmar_drhd_unit *drhd;
2416         struct intel_iommu *iommu;
2417         int nid, ret = 0;
2418
2419         si_domain = alloc_domain(false);
2420         if (!si_domain)
2421                 return -EFAULT;
2422
2423         si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2424
2425         for_each_active_iommu(iommu, drhd) {
2426                 ret = iommu_attach_domain(si_domain, iommu);
2427                 if (ret) {
2428                         domain_exit(si_domain);
2429                         return -EFAULT;
2430                 }
2431         }
2432
2433         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2434                 domain_exit(si_domain);
2435                 return -EFAULT;
2436         }
2437
2438         pr_debug("IOMMU: identity mapping domain is domain %d\n",
2439                  si_domain->id);
2440
2441         if (hw)
2442                 return 0;
2443
2444         for_each_online_node(nid) {
2445                 unsigned long start_pfn, end_pfn;
2446                 int i;
2447
2448                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2449                         ret = iommu_domain_identity_map(si_domain,
2450                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2451                         if (ret)
2452                                 return ret;
2453                 }
2454         }
2455
2456         return 0;
2457 }
2458
2459 static int identity_mapping(struct device *dev)
2460 {
2461         struct device_domain_info *info;
2462
2463         if (likely(!iommu_identity_mapping))
2464                 return 0;
2465
2466         info = dev->archdata.iommu;
2467         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2468                 return (info->domain == si_domain);
2469
2470         return 0;
2471 }
2472
2473 static int domain_add_dev_info(struct dmar_domain *domain,
2474                                struct device *dev, int translation)
2475 {
2476         struct dmar_domain *ndomain;
2477         struct intel_iommu *iommu;
2478         u8 bus, devfn;
2479         int ret;
2480
2481         iommu = device_to_iommu(dev, &bus, &devfn);
2482         if (!iommu)
2483                 return -ENODEV;
2484
2485         ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2486         if (ndomain != domain)
2487                 return -EBUSY;
2488
2489         ret = domain_context_mapping(domain, dev, translation);
2490         if (ret) {
2491                 domain_remove_one_dev_info(domain, dev);
2492                 return ret;
2493         }
2494
2495         return 0;
2496 }
2497
2498 static bool device_has_rmrr(struct device *dev)
2499 {
2500         struct dmar_rmrr_unit *rmrr;
2501         struct device *tmp;
2502         int i;
2503
2504         rcu_read_lock();
2505         for_each_rmrr_units(rmrr) {
2506                 /*
2507                  * Return TRUE if this RMRR contains the device that
2508                  * is passed in.
2509                  */
2510                 for_each_active_dev_scope(rmrr->devices,
2511                                           rmrr->devices_cnt, i, tmp)
2512                         if (tmp == dev) {
2513                                 rcu_read_unlock();
2514                                 return true;
2515                         }
2516         }
2517         rcu_read_unlock();
2518         return false;
2519 }
2520
2521 static int iommu_should_identity_map(struct device *dev, int startup)
2522 {
2523
2524         if (dev_is_pci(dev)) {
2525                 struct pci_dev *pdev = to_pci_dev(dev);
2526
2527                 /*
2528                  * We want to prevent any device associated with an RMRR from
2529                  * getting placed into the SI Domain. This is done because
2530                  * problems exist when devices are moved in and out of domains
2531                  * and their respective RMRR info is lost. We exempt USB devices
2532                  * from this process due to their usage of RMRRs that are known
2533                  * to not be needed after BIOS hand-off to OS.
2534                  */
2535                 if (device_has_rmrr(dev) &&
2536                     (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2537                         return 0;
2538
2539                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2540                         return 1;
2541
2542                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2543                         return 1;
2544
2545                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2546                         return 0;
2547
2548                 /*
2549                  * We want to start off with all devices in the 1:1 domain, and
2550                  * take them out later if we find they can't access all of memory.
2551                  *
2552                  * However, we can't do this for PCI devices behind bridges,
2553                  * because all PCI devices behind the same bridge will end up
2554                  * with the same source-id on their transactions.
2555                  *
2556                  * Practically speaking, we can't change things around for these
2557                  * devices at run-time, because we can't be sure there'll be no
2558                  * DMA transactions in flight for any of their siblings.
2559                  *
2560                  * So PCI devices (unless they're on the root bus) as well as
2561                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2562                  * the 1:1 domain, just in _case_ one of their siblings turns out
2563                  * not to be able to map all of memory.
2564                  */
2565                 if (!pci_is_pcie(pdev)) {
2566                         if (!pci_is_root_bus(pdev->bus))
2567                                 return 0;
2568                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2569                                 return 0;
2570                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2571                         return 0;
2572         } else {
2573                 if (device_has_rmrr(dev))
2574                         return 0;
2575         }
2576
2577         /*
2578          * At boot time, we don't yet know if devices will be 64-bit capable.
2579          * Assume that they will — if they turn out not to be, then we can
2580          * take them out of the 1:1 domain later.
2581          */
2582         if (!startup) {
2583                 /*
2584                  * If the device's dma_mask is less than the system's memory
2585                  * size then this is not a candidate for identity mapping.
2586                  */
2587                 u64 dma_mask = *dev->dma_mask;
2588
2589                 if (dev->coherent_dma_mask &&
2590                     dev->coherent_dma_mask < dma_mask)
2591                         dma_mask = dev->coherent_dma_mask;
2592
2593                 return dma_mask >= dma_get_required_mask(dev);
2594         }
2595
2596         return 1;
2597 }
2598
2599 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2600 {
2601         int ret;
2602
2603         if (!iommu_should_identity_map(dev, 1))
2604                 return 0;
2605
2606         ret = domain_add_dev_info(si_domain, dev,
2607                                   hw ? CONTEXT_TT_PASS_THROUGH :
2608                                        CONTEXT_TT_MULTI_LEVEL);
2609         if (!ret)
2610                 pr_info("IOMMU: %s identity mapping for device %s\n",
2611                         hw ? "hardware" : "software", dev_name(dev));
2612         else if (ret == -ENODEV)
2613                 /* device not associated with an iommu */
2614                 ret = 0;
2615
2616         return ret;
2617 }
2618
2619
2620 static int __init iommu_prepare_static_identity_mapping(int hw)
2621 {
2622         struct pci_dev *pdev = NULL;
2623         struct dmar_drhd_unit *drhd;
2624         struct intel_iommu *iommu;
2625         struct device *dev;
2626         int i;
2627         int ret = 0;
2628
2629         ret = si_domain_init(hw);
2630         if (ret)
2631                 return -EFAULT;
2632
2633         for_each_pci_dev(pdev) {
2634                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2635                 if (ret)
2636                         return ret;
2637         }
2638
2639         for_each_active_iommu(iommu, drhd)
2640                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2641                         struct acpi_device_physical_node *pn;
2642                         struct acpi_device *adev;
2643
2644                         if (dev->bus != &acpi_bus_type)
2645                                 continue;
2646
2647                         adev= to_acpi_device(dev);
2648                         mutex_lock(&adev->physical_node_lock);
2649                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2650                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2651                                 if (ret)
2652                                         break;
2653                         }
2654                         mutex_unlock(&adev->physical_node_lock);
2655                         if (ret)
2656                                 return ret;
2657                 }
2658
2659         return 0;
2660 }
2661
2662 static int __init init_dmars(void)
2663 {
2664         struct dmar_drhd_unit *drhd;
2665         struct dmar_rmrr_unit *rmrr;
2666         struct device *dev;
2667         struct intel_iommu *iommu;
2668         int i, ret;
2669
2670         /*
2671          * for each drhd
2672          *    allocate root
2673          *    initialize and program root entry to not present
2674          * endfor
2675          */
2676         for_each_drhd_unit(drhd) {
2677                 /*
2678                  * lock not needed as this is only incremented in the single
2679                  * threaded kernel __init code path all other access are read
2680                  * only
2681                  */
2682                 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2683                         g_num_of_iommus++;
2684                         continue;
2685                 }
2686                 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2687                           IOMMU_UNITS_SUPPORTED);
2688         }
2689
2690         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2691                         GFP_KERNEL);
2692         if (!g_iommus) {
2693                 printk(KERN_ERR "Allocating global iommu array failed\n");
2694                 ret = -ENOMEM;
2695                 goto error;
2696         }
2697
2698         deferred_flush = kzalloc(g_num_of_iommus *
2699                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2700         if (!deferred_flush) {
2701                 ret = -ENOMEM;
2702                 goto free_g_iommus;
2703         }
2704
2705         for_each_active_iommu(iommu, drhd) {
2706                 g_iommus[iommu->seq_id] = iommu;
2707
2708                 ret = iommu_init_domains(iommu);
2709                 if (ret)
2710                         goto free_iommu;
2711
2712                 /*
2713                  * TBD:
2714                  * we could share the same root & context tables
2715                  * among all IOMMU's. Need to Split it later.
2716                  */
2717                 ret = iommu_alloc_root_entry(iommu);
2718                 if (ret) {
2719                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2720                         goto free_iommu;
2721                 }
2722                 if (!ecap_pass_through(iommu->ecap))
2723                         hw_pass_through = 0;
2724         }
2725
2726         /*
2727          * Start from the sane iommu hardware state.
2728          */
2729         for_each_active_iommu(iommu, drhd) {
2730                 /*
2731                  * If the queued invalidation is already initialized by us
2732                  * (for example, while enabling interrupt-remapping) then
2733                  * we got the things already rolling from a sane state.
2734                  */
2735                 if (iommu->qi)
2736                         continue;
2737
2738                 /*
2739                  * Clear any previous faults.
2740                  */
2741                 dmar_fault(-1, iommu);
2742                 /*
2743                  * Disable queued invalidation if supported and already enabled
2744                  * before OS handover.
2745                  */
2746                 dmar_disable_qi(iommu);
2747         }
2748
2749         for_each_active_iommu(iommu, drhd) {
2750                 if (dmar_enable_qi(iommu)) {
2751                         /*
2752                          * Queued Invalidate not enabled, use Register Based
2753                          * Invalidate
2754                          */
2755                         iommu->flush.flush_context = __iommu_flush_context;
2756                         iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2757                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2758                                "invalidation\n",
2759                                 iommu->seq_id,
2760                                (unsigned long long)drhd->reg_base_addr);
2761                 } else {
2762                         iommu->flush.flush_context = qi_flush_context;
2763                         iommu->flush.flush_iotlb = qi_flush_iotlb;
2764                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2765                                "invalidation\n",
2766                                 iommu->seq_id,
2767                                (unsigned long long)drhd->reg_base_addr);
2768                 }
2769         }
2770
2771         if (iommu_pass_through)
2772                 iommu_identity_mapping |= IDENTMAP_ALL;
2773
2774 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2775         iommu_identity_mapping |= IDENTMAP_GFX;
2776 #endif
2777
2778         check_tylersburg_isoch();
2779
2780         /*
2781          * If pass through is not set or not enabled, setup context entries for
2782          * identity mappings for rmrr, gfx, and isa and may fall back to static
2783          * identity mapping if iommu_identity_mapping is set.
2784          */
2785         if (iommu_identity_mapping) {
2786                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2787                 if (ret) {
2788                         printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2789                         goto free_iommu;
2790                 }
2791         }
2792         /*
2793          * For each rmrr
2794          *   for each dev attached to rmrr
2795          *   do
2796          *     locate drhd for dev, alloc domain for dev
2797          *     allocate free domain
2798          *     allocate page table entries for rmrr
2799          *     if context not allocated for bus
2800          *           allocate and init context
2801          *           set present in root table for this bus
2802          *     init context with domain, translation etc
2803          *    endfor
2804          * endfor
2805          */
2806         printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2807         for_each_rmrr_units(rmrr) {
2808                 /* some BIOS lists non-exist devices in DMAR table. */
2809                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
2810                                           i, dev) {
2811                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
2812                         if (ret)
2813                                 printk(KERN_ERR
2814                                        "IOMMU: mapping reserved region failed\n");
2815                 }
2816         }
2817
2818         iommu_prepare_isa();
2819
2820         /*
2821          * for each drhd
2822          *   enable fault log
2823          *   global invalidate context cache
2824          *   global invalidate iotlb
2825          *   enable translation
2826          */
2827         for_each_iommu(iommu, drhd) {
2828                 if (drhd->ignored) {
2829                         /*
2830                          * we always have to disable PMRs or DMA may fail on
2831                          * this device
2832                          */
2833                         if (force_on)
2834                                 iommu_disable_protect_mem_regions(iommu);
2835                         continue;
2836                 }
2837
2838                 iommu_flush_write_buffer(iommu);
2839
2840                 ret = dmar_set_interrupt(iommu);
2841                 if (ret)
2842                         goto free_iommu;
2843
2844                 iommu_set_root_entry(iommu);
2845
2846                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2847                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2848
2849                 ret = iommu_enable_translation(iommu);
2850                 if (ret)
2851                         goto free_iommu;
2852
2853                 iommu_disable_protect_mem_regions(iommu);
2854         }
2855
2856         return 0;
2857
2858 free_iommu:
2859         for_each_active_iommu(iommu, drhd)
2860                 free_dmar_iommu(iommu);
2861         kfree(deferred_flush);
2862 free_g_iommus:
2863         kfree(g_iommus);
2864 error:
2865         return ret;
2866 }
2867
2868 /* This takes a number of _MM_ pages, not VTD pages */
2869 static struct iova *intel_alloc_iova(struct device *dev,
2870                                      struct dmar_domain *domain,
2871                                      unsigned long nrpages, uint64_t dma_mask)
2872 {
2873         struct iova *iova = NULL;
2874
2875         /* Restrict dma_mask to the width that the iommu can handle */
2876         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2877
2878         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2879                 /*
2880                  * First try to allocate an io virtual address in
2881                  * DMA_BIT_MASK(32) and if that fails then try allocating
2882                  * from higher range
2883                  */
2884                 iova = alloc_iova(&domain->iovad, nrpages,
2885                                   IOVA_PFN(DMA_BIT_MASK(32)), 1);
2886                 if (iova)
2887                         return iova;
2888         }
2889         iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2890         if (unlikely(!iova)) {
2891                 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2892                        nrpages, dev_name(dev));
2893                 return NULL;
2894         }
2895
2896         return iova;
2897 }
2898
2899 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
2900 {
2901         struct dmar_domain *domain;
2902         int ret;
2903
2904         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2905         if (!domain) {
2906                 printk(KERN_ERR "Allocating domain for %s failed",
2907                        dev_name(dev));
2908                 return NULL;
2909         }
2910
2911         /* make sure context mapping is ok */
2912         if (unlikely(!domain_context_mapped(dev))) {
2913                 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2914                 if (ret) {
2915                         printk(KERN_ERR "Domain context map for %s failed",
2916                                dev_name(dev));
2917                         return NULL;
2918                 }
2919         }
2920
2921         return domain;
2922 }
2923
2924 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
2925 {
2926         struct device_domain_info *info;
2927
2928         /* No lock here, assumes no domain exit in normal case */
2929         info = dev->archdata.iommu;
2930         if (likely(info))
2931                 return info->domain;
2932
2933         return __get_valid_domain_for_dev(dev);
2934 }
2935
2936 static int iommu_dummy(struct device *dev)
2937 {
2938         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2939 }
2940
2941 /* Check if the dev needs to go through non-identity map and unmap process.*/
2942 static int iommu_no_mapping(struct device *dev)
2943 {
2944         int found;
2945
2946         if (iommu_dummy(dev))
2947                 return 1;
2948
2949         if (!iommu_identity_mapping)
2950                 return 0;
2951
2952         found = identity_mapping(dev);
2953         if (found) {
2954                 if (iommu_should_identity_map(dev, 0))
2955                         return 1;
2956                 else {
2957                         /*
2958                          * 32 bit DMA is removed from si_domain and fall back
2959                          * to non-identity mapping.
2960                          */
2961                         domain_remove_one_dev_info(si_domain, dev);
2962                         printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2963                                dev_name(dev));
2964                         return 0;
2965                 }
2966         } else {
2967                 /*
2968                  * In case of a detached 64 bit DMA device from vm, the device
2969                  * is put into si_domain for identity mapping.
2970                  */
2971                 if (iommu_should_identity_map(dev, 0)) {
2972                         int ret;
2973                         ret = domain_add_dev_info(si_domain, dev,
2974                                                   hw_pass_through ?
2975                                                   CONTEXT_TT_PASS_THROUGH :
2976                                                   CONTEXT_TT_MULTI_LEVEL);
2977                         if (!ret) {
2978                                 printk(KERN_INFO "64bit %s uses identity mapping\n",
2979                                        dev_name(dev));
2980                                 return 1;
2981                         }
2982                 }
2983         }
2984
2985         return 0;
2986 }
2987
2988 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
2989                                      size_t size, int dir, u64 dma_mask)
2990 {
2991         struct dmar_domain *domain;
2992         phys_addr_t start_paddr;
2993         struct iova *iova;
2994         int prot = 0;
2995         int ret;
2996         struct intel_iommu *iommu;
2997         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2998
2999         BUG_ON(dir == DMA_NONE);
3000
3001         if (iommu_no_mapping(dev))
3002                 return paddr;
3003
3004         domain = get_valid_domain_for_dev(dev);
3005         if (!domain)
3006                 return 0;
3007
3008         iommu = domain_get_iommu(domain);
3009         size = aligned_nrpages(paddr, size);
3010
3011         iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3012         if (!iova)
3013                 goto error;
3014
3015         /*
3016          * Check if DMAR supports zero-length reads on write only
3017          * mappings..
3018          */
3019         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3020                         !cap_zlr(iommu->cap))
3021                 prot |= DMA_PTE_READ;
3022         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3023                 prot |= DMA_PTE_WRITE;
3024         /*
3025          * paddr - (paddr + size) might be partial page, we should map the whole
3026          * page.  Note: if two part of one page are separately mapped, we
3027          * might have two guest_addr mapping to the same host paddr, but this
3028          * is not a big problem
3029          */
3030         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
3031                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3032         if (ret)
3033                 goto error;
3034
3035         /* it's a non-present to present mapping. Only flush if caching mode */
3036         if (cap_caching_mode(iommu->cap))
3037                 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
3038         else
3039                 iommu_flush_write_buffer(iommu);
3040
3041         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3042         start_paddr += paddr & ~PAGE_MASK;
3043         return start_paddr;
3044
3045 error:
3046         if (iova)
3047                 __free_iova(&domain->iovad, iova);
3048         printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
3049                 dev_name(dev), size, (unsigned long long)paddr, dir);
3050         return 0;
3051 }
3052
3053 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3054                                  unsigned long offset, size_t size,
3055                                  enum dma_data_direction dir,
3056                                  struct dma_attrs *attrs)
3057 {
3058         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3059                                   dir, *dev->dma_mask);
3060 }
3061
3062 static void flush_unmaps(void)
3063 {
3064         int i, j;
3065
3066         timer_on = 0;
3067
3068         /* just flush them all */
3069         for (i = 0; i < g_num_of_iommus; i++) {
3070                 struct intel_iommu *iommu = g_iommus[i];
3071                 if (!iommu)
3072                         continue;
3073
3074                 if (!deferred_flush[i].next)
3075                         continue;
3076
3077                 /* In caching mode, global flushes turn emulation expensive */
3078                 if (!cap_caching_mode(iommu->cap))
3079                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3080                                          DMA_TLB_GLOBAL_FLUSH);
3081                 for (j = 0; j < deferred_flush[i].next; j++) {
3082                         unsigned long mask;
3083                         struct iova *iova = deferred_flush[i].iova[j];
3084                         struct dmar_domain *domain = deferred_flush[i].domain[j];
3085
3086                         /* On real hardware multiple invalidations are expensive */
3087                         if (cap_caching_mode(iommu->cap))
3088                                 iommu_flush_iotlb_psi(iommu, domain->id,
3089                                         iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1,
3090                                         !deferred_flush[i].freelist[j], 0);
3091                         else {
3092                                 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
3093                                 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3094                                                 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3095                         }
3096                         __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
3097                         if (deferred_flush[i].freelist[j])
3098                                 dma_free_pagelist(deferred_flush[i].freelist[j]);
3099                 }
3100                 deferred_flush[i].next = 0;
3101         }
3102
3103         list_size = 0;
3104 }
3105
3106 static void flush_unmaps_timeout(unsigned long data)
3107 {
3108         unsigned long flags;
3109
3110         spin_lock_irqsave(&async_umap_flush_lock, flags);
3111         flush_unmaps();
3112         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3113 }
3114
3115 static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
3116 {
3117         unsigned long flags;
3118         int next, iommu_id;
3119         struct intel_iommu *iommu;
3120
3121         spin_lock_irqsave(&async_umap_flush_lock, flags);
3122         if (list_size == HIGH_WATER_MARK)
3123                 flush_unmaps();
3124
3125         iommu = domain_get_iommu(dom);
3126         iommu_id = iommu->seq_id;
3127
3128         next = deferred_flush[iommu_id].next;
3129         deferred_flush[iommu_id].domain[next] = dom;
3130         deferred_flush[iommu_id].iova[next] = iova;
3131         deferred_flush[iommu_id].freelist[next] = freelist;
3132         deferred_flush[iommu_id].next++;
3133
3134         if (!timer_on) {
3135                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3136                 timer_on = 1;
3137         }
3138         list_size++;
3139         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3140 }
3141
3142 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3143                              size_t size, enum dma_data_direction dir,
3144                              struct dma_attrs *attrs)
3145 {
3146         struct dmar_domain *domain;
3147         unsigned long start_pfn, last_pfn;
3148         struct iova *iova;
3149         struct intel_iommu *iommu;
3150         struct page *freelist;
3151
3152         if (iommu_no_mapping(dev))
3153                 return;
3154
3155         domain = find_domain(dev);
3156         BUG_ON(!domain);
3157
3158         iommu = domain_get_iommu(domain);
3159
3160         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
3161         if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3162                       (unsigned long long)dev_addr))
3163                 return;
3164
3165         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3166         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3167
3168         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3169                  dev_name(dev), start_pfn, last_pfn);
3170
3171         freelist = domain_unmap(domain, start_pfn, last_pfn);
3172
3173         if (intel_iommu_strict) {
3174                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3175                                       last_pfn - start_pfn + 1, !freelist, 0);
3176                 /* free iova */
3177                 __free_iova(&domain->iovad, iova);
3178                 dma_free_pagelist(freelist);
3179         } else {
3180                 add_unmap(domain, iova, freelist);
3181                 /*
3182                  * queue up the release of the unmap to save the 1/6th of the
3183                  * cpu used up by the iotlb flush operation...
3184                  */
3185         }
3186 }
3187
3188 static void *intel_alloc_coherent(struct device *dev, size_t size,
3189                                   dma_addr_t *dma_handle, gfp_t flags,
3190                                   struct dma_attrs *attrs)
3191 {
3192         void *vaddr;
3193         int order;
3194
3195         size = PAGE_ALIGN(size);
3196         order = get_order(size);
3197
3198         if (!iommu_no_mapping(dev))
3199                 flags &= ~(GFP_DMA | GFP_DMA32);
3200         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3201                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3202                         flags |= GFP_DMA;
3203                 else
3204                         flags |= GFP_DMA32;
3205         }
3206
3207         vaddr = (void *)__get_free_pages(flags, order);
3208         if (!vaddr)
3209                 return NULL;
3210         memset(vaddr, 0, size);
3211
3212         *dma_handle = __intel_map_single(dev, virt_to_bus(vaddr), size,
3213                                          DMA_BIDIRECTIONAL,
3214                                          dev->coherent_dma_mask);
3215         if (*dma_handle)
3216                 return vaddr;
3217         free_pages((unsigned long)vaddr, order);
3218         return NULL;
3219 }
3220
3221 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3222                                 dma_addr_t dma_handle, struct dma_attrs *attrs)
3223 {
3224         int order;
3225
3226         size = PAGE_ALIGN(size);
3227         order = get_order(size);
3228
3229         intel_unmap_page(dev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
3230         free_pages((unsigned long)vaddr, order);
3231 }
3232
3233 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3234                            int nelems, enum dma_data_direction dir,
3235                            struct dma_attrs *attrs)
3236 {
3237         struct dmar_domain *domain;
3238         unsigned long start_pfn, last_pfn;
3239         struct iova *iova;
3240         struct intel_iommu *iommu;
3241         struct page *freelist;
3242
3243         if (iommu_no_mapping(dev))
3244                 return;
3245
3246         domain = find_domain(dev);
3247         BUG_ON(!domain);
3248
3249         iommu = domain_get_iommu(domain);
3250
3251         iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
3252         if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3253                       (unsigned long long)sglist[0].dma_address))
3254                 return;
3255
3256         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3257         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3258
3259         freelist = domain_unmap(domain, start_pfn, last_pfn);
3260
3261         if (intel_iommu_strict) {
3262                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3263                                       last_pfn - start_pfn + 1, !freelist, 0);
3264                 /* free iova */
3265                 __free_iova(&domain->iovad, iova);
3266                 dma_free_pagelist(freelist);
3267         } else {
3268                 add_unmap(domain, iova, freelist);
3269                 /*
3270                  * queue up the release of the unmap to save the 1/6th of the
3271                  * cpu used up by the iotlb flush operation...
3272                  */
3273         }
3274 }
3275
3276 static int intel_nontranslate_map_sg(struct device *hddev,
3277         struct scatterlist *sglist, int nelems, int dir)
3278 {
3279         int i;
3280         struct scatterlist *sg;
3281
3282         for_each_sg(sglist, sg, nelems, i) {
3283                 BUG_ON(!sg_page(sg));
3284                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3285                 sg->dma_length = sg->length;
3286         }
3287         return nelems;
3288 }
3289
3290 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3291                         enum dma_data_direction dir, struct dma_attrs *attrs)
3292 {
3293         int i;
3294         struct dmar_domain *domain;
3295         size_t size = 0;
3296         int prot = 0;
3297         struct iova *iova = NULL;
3298         int ret;
3299         struct scatterlist *sg;
3300         unsigned long start_vpfn;
3301         struct intel_iommu *iommu;
3302
3303         BUG_ON(dir == DMA_NONE);
3304         if (iommu_no_mapping(dev))
3305                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3306
3307         domain = get_valid_domain_for_dev(dev);
3308         if (!domain)
3309                 return 0;
3310
3311         iommu = domain_get_iommu(domain);
3312
3313         for_each_sg(sglist, sg, nelems, i)
3314                 size += aligned_nrpages(sg->offset, sg->length);
3315
3316         iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3317                                 *dev->dma_mask);
3318         if (!iova) {
3319                 sglist->dma_length = 0;
3320                 return 0;
3321         }
3322
3323         /*
3324          * Check if DMAR supports zero-length reads on write only
3325          * mappings..
3326          */
3327         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3328                         !cap_zlr(iommu->cap))
3329                 prot |= DMA_PTE_READ;
3330         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3331                 prot |= DMA_PTE_WRITE;
3332
3333         start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3334
3335         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3336         if (unlikely(ret)) {
3337                 /*  clear the page */
3338                 dma_pte_clear_range(domain, start_vpfn,
3339                                     start_vpfn + size - 1);
3340                 /* free page tables */
3341                 dma_pte_free_pagetable(domain, start_vpfn,
3342                                        start_vpfn + size - 1);
3343                 /* free iova */
3344                 __free_iova(&domain->iovad, iova);
3345                 return 0;
3346         }
3347
3348         /* it's a non-present to present mapping. Only flush if caching mode */
3349         if (cap_caching_mode(iommu->cap))
3350                 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
3351         else
3352                 iommu_flush_write_buffer(iommu);
3353
3354         return nelems;
3355 }
3356
3357 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3358 {
3359         return !dma_addr;
3360 }
3361
3362 struct dma_map_ops intel_dma_ops = {
3363         .alloc = intel_alloc_coherent,
3364         .free = intel_free_coherent,
3365         .map_sg = intel_map_sg,
3366         .unmap_sg = intel_unmap_sg,
3367         .map_page = intel_map_page,
3368         .unmap_page = intel_unmap_page,
3369         .mapping_error = intel_mapping_error,
3370 };
3371
3372 static inline int iommu_domain_cache_init(void)
3373 {
3374         int ret = 0;
3375
3376         iommu_domain_cache = kmem_cache_create("iommu_domain",
3377                                          sizeof(struct dmar_domain),
3378                                          0,
3379                                          SLAB_HWCACHE_ALIGN,
3380
3381                                          NULL);
3382         if (!iommu_domain_cache) {
3383                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3384                 ret = -ENOMEM;
3385         }
3386
3387         return ret;
3388 }
3389
3390 static inline int iommu_devinfo_cache_init(void)
3391 {
3392         int ret = 0;
3393
3394         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3395                                          sizeof(struct device_domain_info),
3396                                          0,
3397                                          SLAB_HWCACHE_ALIGN,
3398                                          NULL);
3399         if (!iommu_devinfo_cache) {
3400                 printk(KERN_ERR "Couldn't create devinfo cache\n");
3401                 ret = -ENOMEM;
3402         }
3403
3404         return ret;
3405 }
3406
3407 static inline int iommu_iova_cache_init(void)
3408 {
3409         int ret = 0;
3410
3411         iommu_iova_cache = kmem_cache_create("iommu_iova",
3412                                          sizeof(struct iova),
3413                                          0,
3414                                          SLAB_HWCACHE_ALIGN,
3415                                          NULL);
3416         if (!iommu_iova_cache) {
3417                 printk(KERN_ERR "Couldn't create iova cache\n");
3418                 ret = -ENOMEM;
3419         }
3420
3421         return ret;
3422 }
3423
3424 static int __init iommu_init_mempool(void)
3425 {
3426         int ret;
3427         ret = iommu_iova_cache_init();
3428         if (ret)
3429                 return ret;
3430
3431         ret = iommu_domain_cache_init();
3432         if (ret)
3433                 goto domain_error;
3434
3435         ret = iommu_devinfo_cache_init();
3436         if (!ret)
3437                 return ret;
3438
3439         kmem_cache_destroy(iommu_domain_cache);
3440 domain_error:
3441         kmem_cache_destroy(iommu_iova_cache);
3442
3443         return -ENOMEM;
3444 }
3445
3446 static void __init iommu_exit_mempool(void)
3447 {
3448         kmem_cache_destroy(iommu_devinfo_cache);
3449         kmem_cache_destroy(iommu_domain_cache);
3450         kmem_cache_destroy(iommu_iova_cache);
3451
3452 }
3453
3454 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3455 {
3456         struct dmar_drhd_unit *drhd;
3457         u32 vtbar;
3458         int rc;
3459
3460         /* We know that this device on this chipset has its own IOMMU.
3461          * If we find it under a different IOMMU, then the BIOS is lying
3462          * to us. Hope that the IOMMU for this device is actually
3463          * disabled, and it needs no translation...
3464          */
3465         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3466         if (rc) {
3467                 /* "can't" happen */
3468                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3469                 return;
3470         }
3471         vtbar &= 0xffff0000;
3472
3473         /* we know that the this iommu should be at offset 0xa000 from vtbar */
3474         drhd = dmar_find_matched_drhd_unit(pdev);
3475         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3476                             TAINT_FIRMWARE_WORKAROUND,
3477                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3478                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3479 }
3480 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3481
3482 static void __init init_no_remapping_devices(void)
3483 {
3484         struct dmar_drhd_unit *drhd;
3485         struct device *dev;
3486         int i;
3487
3488         for_each_drhd_unit(drhd) {
3489                 if (!drhd->include_all) {
3490                         for_each_active_dev_scope(drhd->devices,
3491                                                   drhd->devices_cnt, i, dev)
3492                                 break;
3493                         /* ignore DMAR unit if no devices exist */
3494                         if (i == drhd->devices_cnt)
3495                                 drhd->ignored = 1;
3496                 }
3497         }
3498
3499         for_each_active_drhd_unit(drhd) {
3500                 if (drhd->include_all)
3501                         continue;
3502
3503                 for_each_active_dev_scope(drhd->devices,
3504                                           drhd->devices_cnt, i, dev)
3505                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3506                                 break;
3507                 if (i < drhd->devices_cnt)
3508                         continue;
3509
3510                 /* This IOMMU has *only* gfx devices. Either bypass it or
3511                    set the gfx_mapped flag, as appropriate */
3512                 if (dmar_map_gfx) {
3513                         intel_iommu_gfx_mapped = 1;
3514                 } else {
3515                         drhd->ignored = 1;
3516                         for_each_active_dev_scope(drhd->devices,
3517                                                   drhd->devices_cnt, i, dev)
3518                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3519                 }
3520         }
3521 }
3522
3523 #ifdef CONFIG_SUSPEND
3524 static int init_iommu_hw(void)
3525 {
3526         struct dmar_drhd_unit *drhd;
3527         struct intel_iommu *iommu = NULL;
3528
3529         for_each_active_iommu(iommu, drhd)
3530                 if (iommu->qi)
3531                         dmar_reenable_qi(iommu);
3532
3533         for_each_iommu(iommu, drhd) {
3534                 if (drhd->ignored) {
3535                         /*
3536                          * we always have to disable PMRs or DMA may fail on
3537                          * this device
3538                          */
3539                         if (force_on)
3540                                 iommu_disable_protect_mem_regions(iommu);
3541                         continue;
3542                 }
3543
3544                 iommu_flush_write_buffer(iommu);
3545
3546                 iommu_set_root_entry(iommu);
3547
3548                 iommu->flush.flush_context(iommu, 0, 0, 0,
3549                                            DMA_CCMD_GLOBAL_INVL);
3550                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3551                                          DMA_TLB_GLOBAL_FLUSH);
3552                 if (iommu_enable_translation(iommu))
3553                         return 1;
3554                 iommu_disable_protect_mem_regions(iommu);
3555         }
3556
3557         return 0;
3558 }
3559
3560 static void iommu_flush_all(void)
3561 {
3562         struct dmar_drhd_unit *drhd;
3563         struct intel_iommu *iommu;
3564
3565         for_each_active_iommu(iommu, drhd) {
3566                 iommu->flush.flush_context(iommu, 0, 0, 0,
3567                                            DMA_CCMD_GLOBAL_INVL);
3568                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3569                                          DMA_TLB_GLOBAL_FLUSH);
3570         }
3571 }
3572
3573 static int iommu_suspend(void)
3574 {
3575         struct dmar_drhd_unit *drhd;
3576         struct intel_iommu *iommu = NULL;
3577         unsigned long flag;
3578
3579         for_each_active_iommu(iommu, drhd) {
3580                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3581                                                  GFP_ATOMIC);
3582                 if (!iommu->iommu_state)
3583                         goto nomem;
3584         }
3585
3586         iommu_flush_all();
3587
3588         for_each_active_iommu(iommu, drhd) {
3589                 iommu_disable_translation(iommu);
3590
3591                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3592
3593                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3594                         readl(iommu->reg + DMAR_FECTL_REG);
3595                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3596                         readl(iommu->reg + DMAR_FEDATA_REG);
3597                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3598                         readl(iommu->reg + DMAR_FEADDR_REG);
3599                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3600                         readl(iommu->reg + DMAR_FEUADDR_REG);
3601
3602                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3603         }
3604         return 0;
3605
3606 nomem:
3607         for_each_active_iommu(iommu, drhd)
3608                 kfree(iommu->iommu_state);
3609
3610         return -ENOMEM;
3611 }
3612
3613 static void iommu_resume(void)
3614 {
3615         struct dmar_drhd_unit *drhd;
3616         struct intel_iommu *iommu = NULL;
3617         unsigned long flag;
3618
3619         if (init_iommu_hw()) {
3620                 if (force_on)
3621                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3622                 else
3623                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3624                 return;
3625         }
3626
3627         for_each_active_iommu(iommu, drhd) {
3628
3629                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3630
3631                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3632                         iommu->reg + DMAR_FECTL_REG);
3633                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3634                         iommu->reg + DMAR_FEDATA_REG);
3635                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3636                         iommu->reg + DMAR_FEADDR_REG);
3637                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3638                         iommu->reg + DMAR_FEUADDR_REG);
3639
3640                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3641         }
3642
3643         for_each_active_iommu(iommu, drhd)
3644                 kfree(iommu->iommu_state);
3645 }
3646
3647 static struct syscore_ops iommu_syscore_ops = {
3648         .resume         = iommu_resume,
3649         .suspend        = iommu_suspend,
3650 };
3651
3652 static void __init init_iommu_pm_ops(void)
3653 {
3654         register_syscore_ops(&iommu_syscore_ops);
3655 }
3656
3657 #else
3658 static inline void init_iommu_pm_ops(void) {}
3659 #endif  /* CONFIG_PM */
3660
3661
3662 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3663 {
3664         struct acpi_dmar_reserved_memory *rmrr;
3665         struct dmar_rmrr_unit *rmrru;
3666
3667         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3668         if (!rmrru)
3669                 return -ENOMEM;
3670
3671         rmrru->hdr = header;
3672         rmrr = (struct acpi_dmar_reserved_memory *)header;
3673         rmrru->base_address = rmrr->base_address;
3674         rmrru->end_address = rmrr->end_address;
3675         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3676                                 ((void *)rmrr) + rmrr->header.length,
3677                                 &rmrru->devices_cnt);
3678         if (rmrru->devices_cnt && rmrru->devices == NULL) {
3679                 kfree(rmrru);
3680                 return -ENOMEM;
3681         }
3682
3683         list_add(&rmrru->list, &dmar_rmrr_units);
3684
3685         return 0;
3686 }
3687
3688 int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3689 {
3690         struct acpi_dmar_atsr *atsr;
3691         struct dmar_atsr_unit *atsru;
3692
3693         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3694         atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3695         if (!atsru)
3696                 return -ENOMEM;
3697
3698         atsru->hdr = hdr;
3699         atsru->include_all = atsr->flags & 0x1;
3700         if (!atsru->include_all) {
3701                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3702                                 (void *)atsr + atsr->header.length,
3703                                 &atsru->devices_cnt);
3704                 if (atsru->devices_cnt && atsru->devices == NULL) {
3705                         kfree(atsru);
3706                         return -ENOMEM;
3707                 }
3708         }
3709
3710         list_add_rcu(&atsru->list, &dmar_atsr_units);
3711
3712         return 0;
3713 }
3714
3715 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3716 {
3717         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3718         kfree(atsru);
3719 }
3720
3721 static void intel_iommu_free_dmars(void)
3722 {
3723         struct dmar_rmrr_unit *rmrru, *rmrr_n;
3724         struct dmar_atsr_unit *atsru, *atsr_n;
3725
3726         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3727                 list_del(&rmrru->list);
3728                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3729                 kfree(rmrru);
3730         }
3731
3732         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3733                 list_del(&atsru->list);
3734                 intel_iommu_free_atsr(atsru);
3735         }
3736 }
3737
3738 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3739 {
3740         int i, ret = 1;
3741         struct pci_bus *bus;
3742         struct pci_dev *bridge = NULL;
3743         struct device *tmp;
3744         struct acpi_dmar_atsr *atsr;
3745         struct dmar_atsr_unit *atsru;
3746
3747         dev = pci_physfn(dev);
3748         for (bus = dev->bus; bus; bus = bus->parent) {
3749                 bridge = bus->self;
3750                 if (!bridge || !pci_is_pcie(bridge) ||
3751                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
3752                         return 0;
3753                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
3754                         break;
3755         }
3756         if (!bridge)
3757                 return 0;
3758
3759         rcu_read_lock();
3760         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3761                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3762                 if (atsr->segment != pci_domain_nr(dev->bus))
3763                         continue;
3764
3765                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
3766                         if (tmp == &bridge->dev)
3767                                 goto out;
3768
3769                 if (atsru->include_all)
3770                         goto out;
3771         }
3772         ret = 0;
3773 out:
3774         rcu_read_unlock();
3775
3776         return ret;
3777 }
3778
3779 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3780 {
3781         int ret = 0;
3782         struct dmar_rmrr_unit *rmrru;
3783         struct dmar_atsr_unit *atsru;
3784         struct acpi_dmar_atsr *atsr;
3785         struct acpi_dmar_reserved_memory *rmrr;
3786
3787         if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
3788                 return 0;
3789
3790         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3791                 rmrr = container_of(rmrru->hdr,
3792                                     struct acpi_dmar_reserved_memory, header);
3793                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3794                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3795                                 ((void *)rmrr) + rmrr->header.length,
3796                                 rmrr->segment, rmrru->devices,
3797                                 rmrru->devices_cnt);
3798                         if (ret > 0)
3799                                 break;
3800                         else if(ret < 0)
3801                                 return ret;
3802                 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3803                         if (dmar_remove_dev_scope(info, rmrr->segment,
3804                                 rmrru->devices, rmrru->devices_cnt))
3805                                 break;
3806                 }
3807         }
3808
3809         list_for_each_entry(atsru, &dmar_atsr_units, list) {
3810                 if (atsru->include_all)
3811                         continue;
3812
3813                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3814                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3815                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3816                                         (void *)atsr + atsr->header.length,
3817                                         atsr->segment, atsru->devices,
3818                                         atsru->devices_cnt);
3819                         if (ret > 0)
3820                                 break;
3821                         else if(ret < 0)
3822                                 return ret;
3823                 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3824                         if (dmar_remove_dev_scope(info, atsr->segment,
3825                                         atsru->devices, atsru->devices_cnt))
3826                                 break;
3827                 }
3828         }
3829
3830         return 0;
3831 }
3832
3833 /*
3834  * Here we only respond to action of unbound device from driver.
3835  *
3836  * Added device is not attached to its DMAR domain here yet. That will happen
3837  * when mapping the device to iova.
3838  */
3839 static int device_notifier(struct notifier_block *nb,
3840                                   unsigned long action, void *data)
3841 {
3842         struct device *dev = data;
3843         struct dmar_domain *domain;
3844
3845         if (iommu_dummy(dev))
3846                 return 0;
3847
3848         if (action != BUS_NOTIFY_UNBOUND_DRIVER &&
3849             action != BUS_NOTIFY_DEL_DEVICE)
3850                 return 0;
3851
3852         domain = find_domain(dev);
3853         if (!domain)
3854                 return 0;
3855
3856         down_read(&dmar_global_lock);
3857         domain_remove_one_dev_info(domain, dev);
3858         if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3859             !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3860             list_empty(&domain->devices))
3861                 domain_exit(domain);
3862         up_read(&dmar_global_lock);
3863
3864         return 0;
3865 }
3866
3867 static struct notifier_block device_nb = {
3868         .notifier_call = device_notifier,
3869 };
3870
3871 static int intel_iommu_memory_notifier(struct notifier_block *nb,
3872                                        unsigned long val, void *v)
3873 {
3874         struct memory_notify *mhp = v;
3875         unsigned long long start, end;
3876         unsigned long start_vpfn, last_vpfn;
3877
3878         switch (val) {
3879         case MEM_GOING_ONLINE:
3880                 start = mhp->start_pfn << PAGE_SHIFT;
3881                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
3882                 if (iommu_domain_identity_map(si_domain, start, end)) {
3883                         pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
3884                                 start, end);
3885                         return NOTIFY_BAD;
3886                 }
3887                 break;
3888
3889         case MEM_OFFLINE:
3890         case MEM_CANCEL_ONLINE:
3891                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
3892                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
3893                 while (start_vpfn <= last_vpfn) {
3894                         struct iova *iova;
3895                         struct dmar_drhd_unit *drhd;
3896                         struct intel_iommu *iommu;
3897                         struct page *freelist;
3898
3899                         iova = find_iova(&si_domain->iovad, start_vpfn);
3900                         if (iova == NULL) {
3901                                 pr_debug("dmar: failed get IOVA for PFN %lx\n",
3902                                          start_vpfn);
3903                                 break;
3904                         }
3905
3906                         iova = split_and_remove_iova(&si_domain->iovad, iova,
3907                                                      start_vpfn, last_vpfn);
3908                         if (iova == NULL) {
3909                                 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
3910                                         start_vpfn, last_vpfn);
3911                                 return NOTIFY_BAD;
3912                         }
3913
3914                         freelist = domain_unmap(si_domain, iova->pfn_lo,
3915                                                iova->pfn_hi);
3916
3917                         rcu_read_lock();
3918                         for_each_active_iommu(iommu, drhd)
3919                                 iommu_flush_iotlb_psi(iommu, si_domain->id,
3920                                         iova->pfn_lo,
3921                                         iova->pfn_hi - iova->pfn_lo + 1,
3922                                         !freelist, 0);
3923                         rcu_read_unlock();
3924                         dma_free_pagelist(freelist);
3925
3926                         start_vpfn = iova->pfn_hi + 1;
3927                         free_iova_mem(iova);
3928                 }
3929                 break;
3930         }
3931
3932         return NOTIFY_OK;
3933 }
3934
3935 static struct notifier_block intel_iommu_memory_nb = {
3936         .notifier_call = intel_iommu_memory_notifier,
3937         .priority = 0
3938 };
3939
3940 int __init intel_iommu_init(void)
3941 {
3942         int ret = -ENODEV;
3943         struct dmar_drhd_unit *drhd;
3944         struct intel_iommu *iommu;
3945
3946         /* VT-d is required for a TXT/tboot launch, so enforce that */
3947         force_on = tboot_force_iommu();
3948
3949         if (iommu_init_mempool()) {
3950                 if (force_on)
3951                         panic("tboot: Failed to initialize iommu memory\n");
3952                 return -ENOMEM;
3953         }
3954
3955         down_write(&dmar_global_lock);
3956         if (dmar_table_init()) {
3957                 if (force_on)
3958                         panic("tboot: Failed to initialize DMAR table\n");
3959                 goto out_free_dmar;
3960         }
3961
3962         /*
3963          * Disable translation if already enabled prior to OS handover.
3964          */
3965         for_each_active_iommu(iommu, drhd)
3966                 if (iommu->gcmd & DMA_GCMD_TE)
3967                         iommu_disable_translation(iommu);
3968
3969         if (dmar_dev_scope_init() < 0) {
3970                 if (force_on)
3971                         panic("tboot: Failed to initialize DMAR device scope\n");
3972                 goto out_free_dmar;
3973         }
3974
3975         if (no_iommu || dmar_disabled)
3976                 goto out_free_dmar;
3977
3978         if (list_empty(&dmar_rmrr_units))
3979                 printk(KERN_INFO "DMAR: No RMRR found\n");
3980
3981         if (list_empty(&dmar_atsr_units))
3982                 printk(KERN_INFO "DMAR: No ATSR found\n");
3983
3984         if (dmar_init_reserved_ranges()) {
3985                 if (force_on)
3986                         panic("tboot: Failed to reserve iommu ranges\n");
3987                 goto out_free_reserved_range;
3988         }
3989
3990         init_no_remapping_devices();
3991
3992         ret = init_dmars();
3993         if (ret) {
3994                 if (force_on)
3995                         panic("tboot: Failed to initialize DMARs\n");
3996                 printk(KERN_ERR "IOMMU: dmar init failed\n");
3997                 goto out_free_reserved_range;
3998         }
3999         up_write(&dmar_global_lock);
4000         printk(KERN_INFO
4001         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
4002
4003         init_timer(&unmap_timer);
4004 #ifdef CONFIG_SWIOTLB
4005         swiotlb = 0;
4006 #endif
4007         dma_ops = &intel_dma_ops;
4008
4009         init_iommu_pm_ops();
4010
4011         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4012         bus_register_notifier(&pci_bus_type, &device_nb);
4013         if (si_domain && !hw_pass_through)
4014                 register_memory_notifier(&intel_iommu_memory_nb);
4015
4016         intel_iommu_enabled = 1;
4017
4018         return 0;
4019
4020 out_free_reserved_range:
4021         put_iova_domain(&reserved_iova_list);
4022 out_free_dmar:
4023         intel_iommu_free_dmars();
4024         up_write(&dmar_global_lock);
4025         iommu_exit_mempool();
4026         return ret;
4027 }
4028
4029 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
4030                                            struct device *dev)
4031 {
4032         struct pci_dev *tmp, *parent, *pdev;
4033
4034         if (!iommu || !dev || !dev_is_pci(dev))
4035                 return;
4036
4037         pdev = to_pci_dev(dev);
4038
4039         /* dependent device detach */
4040         tmp = pci_find_upstream_pcie_bridge(pdev);
4041         /* Secondary interface's bus number and devfn 0 */
4042         if (tmp) {
4043                 parent = pdev->bus->self;
4044                 while (parent != tmp) {
4045                         iommu_detach_dev(iommu, parent->bus->number,
4046                                          parent->devfn);
4047                         parent = parent->bus->self;
4048                 }
4049                 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
4050                         iommu_detach_dev(iommu,
4051                                 tmp->subordinate->number, 0);
4052                 else /* this is a legacy PCI bridge */
4053                         iommu_detach_dev(iommu, tmp->bus->number,
4054                                          tmp->devfn);
4055         }
4056 }
4057
4058 static void domain_remove_one_dev_info(struct dmar_domain *domain,
4059                                        struct device *dev)
4060 {
4061         struct device_domain_info *info, *tmp;
4062         struct intel_iommu *iommu;
4063         unsigned long flags;
4064         int found = 0;
4065         u8 bus, devfn;
4066
4067         iommu = device_to_iommu(dev, &bus, &devfn);
4068         if (!iommu)
4069                 return;
4070
4071         spin_lock_irqsave(&device_domain_lock, flags);
4072         list_for_each_entry_safe(info, tmp, &domain->devices, link) {
4073                 if (info->iommu == iommu && info->bus == bus &&
4074                     info->devfn == devfn) {
4075                         unlink_domain_info(info);
4076                         spin_unlock_irqrestore(&device_domain_lock, flags);
4077
4078                         iommu_disable_dev_iotlb(info);
4079                         iommu_detach_dev(iommu, info->bus, info->devfn);
4080                         iommu_detach_dependent_devices(iommu, dev);
4081                         free_devinfo_mem(info);
4082
4083                         spin_lock_irqsave(&device_domain_lock, flags);
4084
4085                         if (found)
4086                                 break;
4087                         else
4088                                 continue;
4089                 }
4090
4091                 /* if there is no other devices under the same iommu
4092                  * owned by this domain, clear this iommu in iommu_bmp
4093                  * update iommu count and coherency
4094                  */
4095                 if (info->iommu == iommu)
4096                         found = 1;
4097         }
4098
4099         spin_unlock_irqrestore(&device_domain_lock, flags);
4100
4101         if (found == 0) {
4102                 unsigned long tmp_flags;
4103                 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
4104                 clear_bit(iommu->seq_id, domain->iommu_bmp);
4105                 domain->iommu_count--;
4106                 domain_update_iommu_cap(domain);
4107                 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
4108
4109                 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
4110                     !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
4111                         spin_lock_irqsave(&iommu->lock, tmp_flags);
4112                         clear_bit(domain->id, iommu->domain_ids);
4113                         iommu->domains[domain->id] = NULL;
4114                         spin_unlock_irqrestore(&iommu->lock, tmp_flags);
4115                 }
4116         }
4117 }
4118
4119 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4120 {
4121         int adjust_width;
4122
4123         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
4124         domain_reserve_special_ranges(domain);
4125
4126         /* calculate AGAW */
4127         domain->gaw = guest_width;
4128         adjust_width = guestwidth_to_adjustwidth(guest_width);
4129         domain->agaw = width_to_agaw(adjust_width);
4130
4131         domain->iommu_coherency = 0;
4132         domain->iommu_snooping = 0;
4133         domain->iommu_superpage = 0;
4134         domain->max_addr = 0;
4135         domain->nid = -1;
4136
4137         /* always allocate the top pgd */
4138         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4139         if (!domain->pgd)
4140                 return -ENOMEM;
4141         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4142         return 0;
4143 }
4144
4145 static int intel_iommu_domain_init(struct iommu_domain *domain)
4146 {
4147         struct dmar_domain *dmar_domain;
4148
4149         dmar_domain = alloc_domain(true);
4150         if (!dmar_domain) {
4151                 printk(KERN_ERR
4152                         "intel_iommu_domain_init: dmar_domain == NULL\n");
4153                 return -ENOMEM;
4154         }
4155         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4156                 printk(KERN_ERR
4157                         "intel_iommu_domain_init() failed\n");
4158                 domain_exit(dmar_domain);
4159                 return -ENOMEM;
4160         }
4161         domain_update_iommu_cap(dmar_domain);
4162         domain->priv = dmar_domain;
4163
4164         domain->geometry.aperture_start = 0;
4165         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4166         domain->geometry.force_aperture = true;
4167
4168         return 0;
4169 }
4170
4171 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
4172 {
4173         struct dmar_domain *dmar_domain = domain->priv;
4174
4175         domain->priv = NULL;
4176         domain_exit(dmar_domain);
4177 }
4178
4179 static int intel_iommu_attach_device(struct iommu_domain *domain,
4180                                      struct device *dev)
4181 {
4182         struct dmar_domain *dmar_domain = domain->priv;
4183         struct intel_iommu *iommu;
4184         int addr_width;
4185         u8 bus, devfn;
4186
4187         /* normally dev is not mapped */
4188         if (unlikely(domain_context_mapped(dev))) {
4189                 struct dmar_domain *old_domain;
4190
4191                 old_domain = find_domain(dev);
4192                 if (old_domain) {
4193                         if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
4194                             dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
4195                                 domain_remove_one_dev_info(old_domain, dev);
4196                         else
4197                                 domain_remove_dev_info(old_domain);
4198                 }
4199         }
4200
4201         iommu = device_to_iommu(dev, &bus, &devfn);
4202         if (!iommu)
4203                 return -ENODEV;
4204
4205         /* check if this iommu agaw is sufficient for max mapped address */
4206         addr_width = agaw_to_width(iommu->agaw);
4207         if (addr_width > cap_mgaw(iommu->cap))
4208                 addr_width = cap_mgaw(iommu->cap);
4209
4210         if (dmar_domain->max_addr > (1LL << addr_width)) {
4211                 printk(KERN_ERR "%s: iommu width (%d) is not "
4212                        "sufficient for the mapped address (%llx)\n",
4213                        __func__, addr_width, dmar_domain->max_addr);
4214                 return -EFAULT;
4215         }
4216         dmar_domain->gaw = addr_width;
4217
4218         /*
4219          * Knock out extra levels of page tables if necessary
4220          */
4221         while (iommu->agaw < dmar_domain->agaw) {
4222                 struct dma_pte *pte;
4223
4224                 pte = dmar_domain->pgd;
4225                 if (dma_pte_present(pte)) {
4226                         dmar_domain->pgd = (struct dma_pte *)
4227                                 phys_to_virt(dma_pte_addr(pte));
4228                         free_pgtable_page(pte);
4229                 }
4230                 dmar_domain->agaw--;
4231         }
4232
4233         return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
4234 }
4235
4236 static void intel_iommu_detach_device(struct iommu_domain *domain,
4237                                       struct device *dev)
4238 {
4239         struct dmar_domain *dmar_domain = domain->priv;
4240
4241         domain_remove_one_dev_info(dmar_domain, dev);
4242 }
4243
4244 static int intel_iommu_map(struct iommu_domain *domain,
4245                            unsigned long iova, phys_addr_t hpa,
4246                            size_t size, int iommu_prot)
4247 {
4248         struct dmar_domain *dmar_domain = domain->priv;
4249         u64 max_addr;
4250         int prot = 0;
4251         int ret;
4252
4253         if (iommu_prot & IOMMU_READ)
4254                 prot |= DMA_PTE_READ;
4255         if (iommu_prot & IOMMU_WRITE)
4256                 prot |= DMA_PTE_WRITE;
4257         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4258                 prot |= DMA_PTE_SNP;
4259
4260         max_addr = iova + size;
4261         if (dmar_domain->max_addr < max_addr) {
4262                 u64 end;
4263
4264                 /* check if minimum agaw is sufficient for mapped address */
4265                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4266                 if (end < max_addr) {
4267                         printk(KERN_ERR "%s: iommu width (%d) is not "
4268                                "sufficient for the mapped address (%llx)\n",
4269                                __func__, dmar_domain->gaw, max_addr);
4270                         return -EFAULT;
4271                 }
4272                 dmar_domain->max_addr = max_addr;
4273         }
4274         /* Round up size to next multiple of PAGE_SIZE, if it and
4275            the low bits of hpa would take us onto the next page */
4276         size = aligned_nrpages(hpa, size);
4277         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4278                                  hpa >> VTD_PAGE_SHIFT, size, prot);
4279         return ret;
4280 }
4281
4282 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4283                                 unsigned long iova, size_t size)
4284 {
4285         struct dmar_domain *dmar_domain = domain->priv;
4286         struct page *freelist = NULL;
4287         struct intel_iommu *iommu;
4288         unsigned long start_pfn, last_pfn;
4289         unsigned int npages;
4290         int iommu_id, num, ndomains, level = 0;
4291
4292         /* Cope with horrid API which requires us to unmap more than the
4293            size argument if it happens to be a large-page mapping. */
4294         if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4295                 BUG();
4296
4297         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4298                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4299
4300         start_pfn = iova >> VTD_PAGE_SHIFT;
4301         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4302
4303         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4304
4305         npages = last_pfn - start_pfn + 1;
4306
4307         for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4308                iommu = g_iommus[iommu_id];
4309
4310                /*
4311                 * find bit position of dmar_domain
4312                 */
4313                ndomains = cap_ndoms(iommu->cap);
4314                for_each_set_bit(num, iommu->domain_ids, ndomains) {
4315                        if (iommu->domains[num] == dmar_domain)
4316                                iommu_flush_iotlb_psi(iommu, num, start_pfn,
4317                                                      npages, !freelist, 0);
4318                }
4319
4320         }
4321
4322         dma_free_pagelist(freelist);
4323
4324         if (dmar_domain->max_addr == iova + size)
4325                 dmar_domain->max_addr = iova;
4326
4327         return size;
4328 }
4329
4330 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4331                                             dma_addr_t iova)
4332 {
4333         struct dmar_domain *dmar_domain = domain->priv;
4334         struct dma_pte *pte;
4335         int level = 0;
4336         u64 phys = 0;
4337
4338         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
4339         if (pte)
4340                 phys = dma_pte_addr(pte);
4341
4342         return phys;
4343 }
4344
4345 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4346                                       unsigned long cap)
4347 {
4348         struct dmar_domain *dmar_domain = domain->priv;
4349
4350         if (cap == IOMMU_CAP_CACHE_COHERENCY)
4351                 return dmar_domain->iommu_snooping;
4352         if (cap == IOMMU_CAP_INTR_REMAP)
4353                 return irq_remapping_enabled;
4354
4355         return 0;
4356 }
4357
4358 #define REQ_ACS_FLAGS   (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
4359
4360 static int intel_iommu_add_device(struct device *dev)
4361 {
4362         struct pci_dev *pdev = to_pci_dev(dev);
4363         struct pci_dev *bridge, *dma_pdev = NULL;
4364         struct iommu_group *group;
4365         int ret;
4366         u8 bus, devfn;
4367
4368         if (!device_to_iommu(dev, &bus, &devfn))
4369                 return -ENODEV;
4370
4371         bridge = pci_find_upstream_pcie_bridge(pdev);
4372         if (bridge) {
4373                 if (pci_is_pcie(bridge))
4374                         dma_pdev = pci_get_domain_bus_and_slot(
4375                                                 pci_domain_nr(pdev->bus),
4376                                                 bridge->subordinate->number, 0);
4377                 if (!dma_pdev)
4378                         dma_pdev = pci_dev_get(bridge);
4379         } else
4380                 dma_pdev = pci_dev_get(pdev);
4381
4382         /* Account for quirked devices */
4383         swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
4384
4385         /*
4386          * If it's a multifunction device that does not support our
4387          * required ACS flags, add to the same group as lowest numbered
4388          * function that also does not suport the required ACS flags.
4389          */
4390         if (dma_pdev->multifunction &&
4391             !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
4392                 u8 i, slot = PCI_SLOT(dma_pdev->devfn);
4393
4394                 for (i = 0; i < 8; i++) {
4395                         struct pci_dev *tmp;
4396
4397                         tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
4398                         if (!tmp)
4399                                 continue;
4400
4401                         if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
4402                                 swap_pci_ref(&dma_pdev, tmp);
4403                                 break;
4404                         }
4405                         pci_dev_put(tmp);
4406                 }
4407         }
4408
4409         /*
4410          * Devices on the root bus go through the iommu.  If that's not us,
4411          * find the next upstream device and test ACS up to the root bus.
4412          * Finding the next device may require skipping virtual buses.
4413          */
4414         while (!pci_is_root_bus(dma_pdev->bus)) {
4415                 struct pci_bus *bus = dma_pdev->bus;
4416
4417                 while (!bus->self) {
4418                         if (!pci_is_root_bus(bus))
4419                                 bus = bus->parent;
4420                         else
4421                                 goto root_bus;
4422                 }
4423
4424                 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
4425                         break;
4426
4427                 swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
4428         }
4429
4430 root_bus:
4431         group = iommu_group_get(&dma_pdev->dev);
4432         pci_dev_put(dma_pdev);
4433         if (!group) {
4434                 group = iommu_group_alloc();
4435                 if (IS_ERR(group))
4436                         return PTR_ERR(group);
4437         }
4438
4439         ret = iommu_group_add_device(group, dev);
4440
4441         iommu_group_put(group);
4442         return ret;
4443 }
4444
4445 static void intel_iommu_remove_device(struct device *dev)
4446 {
4447         iommu_group_remove_device(dev);
4448 }
4449
4450 static struct iommu_ops intel_iommu_ops = {
4451         .domain_init    = intel_iommu_domain_init,
4452         .domain_destroy = intel_iommu_domain_destroy,
4453         .attach_dev     = intel_iommu_attach_device,
4454         .detach_dev     = intel_iommu_detach_device,
4455         .map            = intel_iommu_map,
4456         .unmap          = intel_iommu_unmap,
4457         .iova_to_phys   = intel_iommu_iova_to_phys,
4458         .domain_has_cap = intel_iommu_domain_has_cap,
4459         .add_device     = intel_iommu_add_device,
4460         .remove_device  = intel_iommu_remove_device,
4461         .pgsize_bitmap  = INTEL_IOMMU_PGSIZES,
4462 };
4463
4464 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4465 {
4466         /* G4x/GM45 integrated gfx dmar support is totally busted. */
4467         printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4468         dmar_map_gfx = 0;
4469 }
4470
4471 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4472 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4473 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4474 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4475 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4476 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4477 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4478
4479 static void quirk_iommu_rwbf(struct pci_dev *dev)
4480 {
4481         /*
4482          * Mobile 4 Series Chipset neglects to set RWBF capability,
4483          * but needs it. Same seems to hold for the desktop versions.
4484          */
4485         printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4486         rwbf_quirk = 1;
4487 }
4488
4489 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4490 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4491 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4492 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4493 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4494 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4495 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4496
4497 #define GGC 0x52
4498 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
4499 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
4500 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
4501 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
4502 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
4503 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
4504 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
4505 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
4506
4507 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4508 {
4509         unsigned short ggc;
4510
4511         if (pci_read_config_word(dev, GGC, &ggc))
4512                 return;
4513
4514         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4515                 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4516                 dmar_map_gfx = 0;
4517         } else if (dmar_map_gfx) {
4518                 /* we have to ensure the gfx device is idle before we flush */
4519                 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4520                 intel_iommu_strict = 1;
4521        }
4522 }
4523 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4524 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4525 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4526 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4527
4528 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4529    ISOCH DMAR unit for the Azalia sound device, but not give it any
4530    TLB entries, which causes it to deadlock. Check for that.  We do
4531    this in a function called from init_dmars(), instead of in a PCI
4532    quirk, because we don't want to print the obnoxious "BIOS broken"
4533    message if VT-d is actually disabled.
4534 */
4535 static void __init check_tylersburg_isoch(void)
4536 {
4537         struct pci_dev *pdev;
4538         uint32_t vtisochctrl;
4539
4540         /* If there's no Azalia in the system anyway, forget it. */
4541         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4542         if (!pdev)
4543                 return;
4544         pci_dev_put(pdev);
4545
4546         /* System Management Registers. Might be hidden, in which case
4547            we can't do the sanity check. But that's OK, because the
4548            known-broken BIOSes _don't_ actually hide it, so far. */
4549         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4550         if (!pdev)
4551                 return;
4552
4553         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4554                 pci_dev_put(pdev);
4555                 return;
4556         }
4557
4558         pci_dev_put(pdev);
4559
4560         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4561         if (vtisochctrl & 1)
4562                 return;
4563
4564         /* Drop all bits other than the number of TLB entries */
4565         vtisochctrl &= 0x1c;
4566
4567         /* If we have the recommended number of TLB entries (16), fine. */
4568         if (vtisochctrl == 0x10)
4569                 return;
4570
4571         /* Zero TLB entries? You get to ride the short bus to school. */
4572         if (!vtisochctrl) {
4573                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4574                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4575                      dmi_get_system_info(DMI_BIOS_VENDOR),
4576                      dmi_get_system_info(DMI_BIOS_VERSION),
4577                      dmi_get_system_info(DMI_PRODUCT_VERSION));
4578                 iommu_identity_mapping |= IDENTMAP_AZALIA;
4579                 return;
4580         }
4581
4582         printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4583                vtisochctrl);
4584 }