drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  *      - Extended Stream ID (16 bit)
  28  */
  29
  30 #define pr_fmt(fmt) "arm-smmu: " fmt
  31
  32 #include <linux/acpi.h>
  33 #include <linux/acpi_iort.h>
  34 #include <linux/atomic.h>
  35 #include <linux/delay.h>
  36 #include <linux/dma-iommu.h>
  37 #include <linux/dma-mapping.h>
  38 #include <linux/err.h>
  39 #include <linux/interrupt.h>
  40 #include <linux/io.h>
  41 #include <linux/io-64-nonatomic-hi-lo.h>
  42 #include <linux/iommu.h>
  43 #include <linux/iopoll.h>
  44 #include <linux/module.h>
  45 #include <linux/of.h>
  46 #include <linux/of_address.h>
  47 #include <linux/of_device.h>
  48 #include <linux/of_iommu.h>
  49 #include <linux/pci.h>
  50 #include <linux/platform_device.h>
  51 #include <linux/slab.h>
  52 #include <linux/spinlock.h>
  53
  54 #include <linux/amba/bus.h>
  55
  56 #include "io-pgtable.h"
  57
  58 /* Maximum number of context banks per SMMU */
  59 #define ARM_SMMU_MAX_CBS                128
  60
  61 /* SMMU global address space */
  62 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  63 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  64
  65 /*
  66  * SMMU global address space with conditional offset to access secure
  67  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  68  * nsGFSYNR0: 0x450)
  69  */
  70 #define ARM_SMMU_GR0_NS(smmu)                                           \
  71         ((smmu)->base +                                                 \
  72                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  73                         ? 0x400 : 0))
  74
  75 /*
  76  * Some 64-bit registers only make sense to write atomically, but in such
  77  * cases all the data relevant to AArch32 formats lies within the lower word,
  78  * therefore this actually makes more sense than it might first appear.
  79  */
  80 #ifdef CONFIG_64BIT
  81 #define smmu_write_atomic_lq            writeq_relaxed
  82 #else
  83 #define smmu_write_atomic_lq            writel_relaxed
  84 #endif
  85
  86 /* Configuration registers */
  87 #define ARM_SMMU_GR0_sCR0               0x0
  88 #define sCR0_CLIENTPD                   (1 << 0)
  89 #define sCR0_GFRE                       (1 << 1)
  90 #define sCR0_GFIE                       (1 << 2)
  91 #define sCR0_EXIDENABLE                 (1 << 3)
  92 #define sCR0_GCFGFRE                    (1 << 4)
  93 #define sCR0_GCFGFIE                    (1 << 5)
  94 #define sCR0_USFCFG                     (1 << 10)
  95 #define sCR0_VMIDPNE                    (1 << 11)
  96 #define sCR0_PTM                        (1 << 12)
  97 #define sCR0_FB                         (1 << 13)
  98 #define sCR0_VMID16EN                   (1 << 31)
  99 #define sCR0_BSU_SHIFT                  14
 100 #define sCR0_BSU_MASK                   0x3
 101
 102 /* Auxiliary Configuration register */
 103 #define ARM_SMMU_GR0_sACR               0x10
 104
 105 /* Identification registers */
 106 #define ARM_SMMU_GR0_ID0                0x20
 107 #define ARM_SMMU_GR0_ID1                0x24
 108 #define ARM_SMMU_GR0_ID2                0x28
 109 #define ARM_SMMU_GR0_ID3                0x2c
 110 #define ARM_SMMU_GR0_ID4                0x30
 111 #define ARM_SMMU_GR0_ID5                0x34
 112 #define ARM_SMMU_GR0_ID6                0x38
 113 #define ARM_SMMU_GR0_ID7                0x3c
 114 #define ARM_SMMU_GR0_sGFSR              0x48
 115 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 116 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 117 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 118
 119 #define ID0_S1TS                        (1 << 30)
 120 #define ID0_S2TS                        (1 << 29)
 121 #define ID0_NTS                         (1 << 28)
 122 #define ID0_SMS                         (1 << 27)
 123 #define ID0_ATOSNS                      (1 << 26)
 124 #define ID0_PTFS_NO_AARCH32             (1 << 25)
 125 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
 126 #define ID0_CTTW                        (1 << 14)
 127 #define ID0_NUMIRPT_SHIFT               16
 128 #define ID0_NUMIRPT_MASK                0xff
 129 #define ID0_NUMSIDB_SHIFT               9
 130 #define ID0_NUMSIDB_MASK                0xf
 131 #define ID0_EXIDS                       (1 << 8)
 132 #define ID0_NUMSMRG_SHIFT               0
 133 #define ID0_NUMSMRG_MASK                0xff
 134
 135 #define ID1_PAGESIZE                    (1 << 31)
 136 #define ID1_NUMPAGENDXB_SHIFT           28
 137 #define ID1_NUMPAGENDXB_MASK            7
 138 #define ID1_NUMS2CB_SHIFT               16
 139 #define ID1_NUMS2CB_MASK                0xff
 140 #define ID1_NUMCB_SHIFT                 0
 141 #define ID1_NUMCB_MASK                  0xff
 142
 143 #define ID2_OAS_SHIFT                   4
 144 #define ID2_OAS_MASK                    0xf
 145 #define ID2_IAS_SHIFT                   0
 146 #define ID2_IAS_MASK                    0xf
 147 #define ID2_UBS_SHIFT                   8
 148 #define ID2_UBS_MASK                    0xf
 149 #define ID2_PTFS_4K                     (1 << 12)
 150 #define ID2_PTFS_16K                    (1 << 13)
 151 #define ID2_PTFS_64K                    (1 << 14)
 152 #define ID2_VMID16                      (1 << 15)
 153
 154 #define ID7_MAJOR_SHIFT                 4
 155 #define ID7_MAJOR_MASK                  0xf
 156
 157 /* Global TLB invalidation */
 158 #define ARM_SMMU_GR0_TLBIVMID           0x64
 159 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 160 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 161 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 162 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 163 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 164 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 165
 166 /* Stream mapping registers */
 167 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 168 #define SMR_VALID                       (1 << 31)
 169 #define SMR_MASK_SHIFT                  16
 170 #define SMR_ID_SHIFT                    0
 171
 172 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 173 #define S2CR_CBNDX_SHIFT                0
 174 #define S2CR_CBNDX_MASK                 0xff
 175 #define S2CR_EXIDVALID                  (1 << 10)
 176 #define S2CR_TYPE_SHIFT                 16
 177 #define S2CR_TYPE_MASK                  0x3
 178 enum arm_smmu_s2cr_type {
 179         S2CR_TYPE_TRANS,
 180         S2CR_TYPE_BYPASS,
 181         S2CR_TYPE_FAULT,
 182 };
 183
 184 #define S2CR_PRIVCFG_SHIFT              24
 185 #define S2CR_PRIVCFG_MASK               0x3
 186 enum arm_smmu_s2cr_privcfg {
 187         S2CR_PRIVCFG_DEFAULT,
 188         S2CR_PRIVCFG_DIPAN,
 189         S2CR_PRIVCFG_UNPRIV,
 190         S2CR_PRIVCFG_PRIV,
 191 };
 192
 193 /* Context bank attribute registers */
 194 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 195 #define CBAR_VMID_SHIFT                 0
 196 #define CBAR_VMID_MASK                  0xff
 197 #define CBAR_S1_BPSHCFG_SHIFT           8
 198 #define CBAR_S1_BPSHCFG_MASK            3
 199 #define CBAR_S1_BPSHCFG_NSH             3
 200 #define CBAR_S1_MEMATTR_SHIFT           12
 201 #define CBAR_S1_MEMATTR_MASK            0xf
 202 #define CBAR_S1_MEMATTR_WB              0xf
 203 #define CBAR_TYPE_SHIFT                 16
 204 #define CBAR_TYPE_MASK                  0x3
 205 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 206 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 207 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 208 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 209 #define CBAR_IRPTNDX_SHIFT              24
 210 #define CBAR_IRPTNDX_MASK               0xff
 211
 212 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 213 #define CBA2R_RW64_32BIT                (0 << 0)
 214 #define CBA2R_RW64_64BIT                (1 << 0)
 215 #define CBA2R_VMID_SHIFT                16
 216 #define CBA2R_VMID_MASK                 0xffff
 217
 218 /* Translation context bank */
 219 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
 220 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
 221
 222 #define ARM_SMMU_CB_SCTLR               0x0
 223 #define ARM_SMMU_CB_ACTLR               0x4
 224 #define ARM_SMMU_CB_RESUME              0x8
 225 #define ARM_SMMU_CB_TTBCR2              0x10
 226 #define ARM_SMMU_CB_TTBR0               0x20
 227 #define ARM_SMMU_CB_TTBR1               0x28
 228 #define ARM_SMMU_CB_TTBCR               0x30
 229 #define ARM_SMMU_CB_CONTEXTIDR          0x34
 230 #define ARM_SMMU_CB_S1_MAIR0            0x38
 231 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 232 #define ARM_SMMU_CB_PAR                 0x50
 233 #define ARM_SMMU_CB_FSR                 0x58
 234 #define ARM_SMMU_CB_FAR                 0x60
 235 #define ARM_SMMU_CB_FSYNR0              0x68
 236 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 237 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 238 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 239 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 240 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 241 #define ARM_SMMU_CB_ATS1PR              0x800
 242 #define ARM_SMMU_CB_ATSR                0x8f0
 243
 244 #define SCTLR_S1_ASIDPNE                (1 << 12)
 245 #define SCTLR_CFCFG                     (1 << 7)
 246 #define SCTLR_CFIE                      (1 << 6)
 247 #define SCTLR_CFRE                      (1 << 5)
 248 #define SCTLR_E                         (1 << 4)
 249 #define SCTLR_AFE                       (1 << 2)
 250 #define SCTLR_TRE                       (1 << 1)
 251 #define SCTLR_M                         (1 << 0)
 252
 253 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 254
 255 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 256 #define ARM_MMU500_ACR_SMTNMB_TLBEN     (1 << 8)
 257
 258 #define CB_PAR_F                        (1 << 0)
 259
 260 #define ATSR_ACTIVE                     (1 << 0)
 261
 262 #define RESUME_RETRY                    (0 << 0)
 263 #define RESUME_TERMINATE                (1 << 0)
 264
 265 #define TTBCR2_SEP_SHIFT                15
 266 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 267 #define TTBCR2_AS                       (1 << 4)
 268
 269 #define TTBRn_ASID_SHIFT                48
 270
 271 #define FSR_MULTI                       (1 << 31)
 272 #define FSR_SS                          (1 << 30)
 273 #define FSR_UUT                         (1 << 8)
 274 #define FSR_ASF                         (1 << 7)
 275 #define FSR_TLBLKF                      (1 << 6)
 276 #define FSR_TLBMCF                      (1 << 5)
 277 #define FSR_EF                          (1 << 4)
 278 #define FSR_PF                          (1 << 3)
 279 #define FSR_AFF                         (1 << 2)
 280 #define FSR_TF                          (1 << 1)
 281
 282 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 283                                          FSR_TLBMCF | FSR_TLBLKF)
 284 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 285                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 286
 287 #define FSYNR0_WNR                      (1 << 4)
 288
 289 #define MSI_IOVA_BASE                   0x8000000
 290 #define MSI_IOVA_LENGTH                 0x100000
 291
 292 static int force_stage;
 293 module_param(force_stage, int, S_IRUGO);
 294 MODULE_PARM_DESC(force_stage,
 295         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 296 static bool disable_bypass;
 297 module_param(disable_bypass, bool, S_IRUGO);
 298 MODULE_PARM_DESC(disable_bypass,
 299         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 300
 301 enum arm_smmu_arch_version {
 302         ARM_SMMU_V1,
 303         ARM_SMMU_V1_64K,
 304         ARM_SMMU_V2,
 305 };
 306
 307 enum arm_smmu_implementation {
 308         GENERIC_SMMU,
 309         ARM_MMU500,
 310         CAVIUM_SMMUV2,
 311 };
 312
 313 struct arm_smmu_s2cr {
 314         struct iommu_group              *group;
 315         int                             count;
 316         enum arm_smmu_s2cr_type         type;
 317         enum arm_smmu_s2cr_privcfg      privcfg;
 318         u8                              cbndx;
 319 };
 320
 321 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 322         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 323 }
 324
 325 struct arm_smmu_smr {
 326         u16                             mask;
 327         u16                             id;
 328         bool                            valid;
 329 };
 330
 331 struct arm_smmu_master_cfg {
 332         struct arm_smmu_device          *smmu;
 333         s16                             smendx[];
 334 };
 335 #define INVALID_SMENDX                  -1
 336 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 337 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 338 #define fwspec_smendx(fw, i) \
 339         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 340 #define for_each_cfg_sme(fw, i, idx) \
 341         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 342
 343 struct arm_smmu_device {
 344         struct device                   *dev;
 345
 346         void __iomem                    *base;
 347         unsigned long                   size;
 348         unsigned long                   pgshift;
 349
 350 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 351 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 352 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 353 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 354 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 355 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 356 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 357 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 358 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 359 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 360 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 361 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 362 #define ARM_SMMU_FEAT_EXIDS             (1 << 12)
 363         u32                             features;
 364
 365 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 366         u32                             options;
 367         enum arm_smmu_arch_version      version;
 368         enum arm_smmu_implementation    model;
 369
 370         u32                             num_context_banks;
 371         u32                             num_s2_context_banks;
 372         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 373         atomic_t                        irptndx;
 374
 375         u32                             num_mapping_groups;
 376         u16                             streamid_mask;
 377         u16                             smr_mask_mask;
 378         struct arm_smmu_smr             *smrs;
 379         struct arm_smmu_s2cr            *s2crs;
 380         struct mutex                    stream_map_mutex;
 381
 382         unsigned long                   va_size;
 383         unsigned long                   ipa_size;
 384         unsigned long                   pa_size;
 385         unsigned long                   pgsize_bitmap;
 386
 387         u32                             num_global_irqs;
 388         u32                             num_context_irqs;
 389         unsigned int                    *irqs;
 390
 391         u32                             cavium_id_base; /* Specific to Cavium */
 392
 393         /* IOMMU core code handle */
 394         struct iommu_device             iommu;
 395 };
 396
 397 enum arm_smmu_context_fmt {
 398         ARM_SMMU_CTX_FMT_NONE,
 399         ARM_SMMU_CTX_FMT_AARCH64,
 400         ARM_SMMU_CTX_FMT_AARCH32_L,
 401         ARM_SMMU_CTX_FMT_AARCH32_S,
 402 };
 403
 404 struct arm_smmu_cfg {
 405         u8                              cbndx;
 406         u8                              irptndx;
 407         u32                             cbar;
 408         enum arm_smmu_context_fmt       fmt;
 409 };
 410 #define INVALID_IRPTNDX                 0xff
 411
 412 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
 413 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
 414
 415 enum arm_smmu_domain_stage {
 416         ARM_SMMU_DOMAIN_S1 = 0,
 417         ARM_SMMU_DOMAIN_S2,
 418         ARM_SMMU_DOMAIN_NESTED,
 419 };
 420
 421 struct arm_smmu_domain {
 422         struct arm_smmu_device          *smmu;
 423         struct io_pgtable_ops           *pgtbl_ops;
 424         spinlock_t                      pgtbl_lock;
 425         struct arm_smmu_cfg             cfg;
 426         enum arm_smmu_domain_stage      stage;
 427         struct mutex                    init_mutex; /* Protects smmu pointer */
 428         struct iommu_domain             domain;
 429 };
 430
 431 struct arm_smmu_option_prop {
 432         u32 opt;
 433         const char *prop;
 434 };
 435
 436 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 437
 438 static bool using_legacy_binding, using_generic_binding;
 439
 440 static struct arm_smmu_option_prop arm_smmu_options[] = {
 441         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 442         { 0, NULL},
 443 };
 444
 445 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 446 {
 447         return container_of(dom, struct arm_smmu_domain, domain);
 448 }
 449
 450 static void parse_driver_options(struct arm_smmu_device *smmu)
 451 {
 452         int i = 0;
 453
 454         do {
 455                 if (of_property_read_bool(smmu->dev->of_node,
 456                                                 arm_smmu_options[i].prop)) {
 457                         smmu->options |= arm_smmu_options[i].opt;
 458                         dev_notice(smmu->dev, "option %s\n",
 459                                 arm_smmu_options[i].prop);
 460                 }
 461         } while (arm_smmu_options[++i].opt);
 462 }
 463
 464 static struct device_node *dev_get_dev_node(struct device *dev)
 465 {
 466         if (dev_is_pci(dev)) {
 467                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 468
 469                 while (!pci_is_root_bus(bus))
 470                         bus = bus->parent;
 471                 return of_node_get(bus->bridge->parent->of_node);
 472         }
 473
 474         return of_node_get(dev->of_node);
 475 }
 476
 477 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 478 {
 479         *((__be32 *)data) = cpu_to_be32(alias);
 480         return 0; /* Continue walking */
 481 }
 482
 483 static int __find_legacy_master_phandle(struct device *dev, void *data)
 484 {
 485         struct of_phandle_iterator *it = *(void **)data;
 486         struct device_node *np = it->node;
 487         int err;
 488
 489         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 490                             "#stream-id-cells", 0)
 491                 if (it->node == np) {
 492                         *(void **)data = dev;
 493                         return 1;
 494                 }
 495         it->node = np;
 496         return err == -ENOENT ? 0 : err;
 497 }
 498
 499 static struct platform_driver arm_smmu_driver;
 500 static struct iommu_ops arm_smmu_ops;
 501
 502 static int arm_smmu_register_legacy_master(struct device *dev,
 503                                            struct arm_smmu_device **smmu)
 504 {
 505         struct device *smmu_dev;
 506         struct device_node *np;
 507         struct of_phandle_iterator it;
 508         void *data = &it;
 509         u32 *sids;
 510         __be32 pci_sid;
 511         int err;
 512
 513         np = dev_get_dev_node(dev);
 514         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 515                 of_node_put(np);
 516                 return -ENODEV;
 517         }
 518
 519         it.node = np;
 520         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 521                                      __find_legacy_master_phandle);
 522         smmu_dev = data;
 523         of_node_put(np);
 524         if (err == 0)
 525                 return -ENODEV;
 526         if (err < 0)
 527                 return err;
 528
 529         if (dev_is_pci(dev)) {
 530                 /* "mmu-masters" assumes Stream ID == Requester ID */
 531                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 532                                        &pci_sid);
 533                 it.cur = &pci_sid;
 534                 it.cur_count = 1;
 535         }
 536
 537         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 538                                 &arm_smmu_ops);
 539         if (err)
 540                 return err;
 541
 542         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 543         if (!sids)
 544                 return -ENOMEM;
 545
 546         *smmu = dev_get_drvdata(smmu_dev);
 547         of_phandle_iterator_args(&it, sids, it.cur_count);
 548         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 549         kfree(sids);
 550         return err;
 551 }
 552
 553 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 554 {
 555         int idx;
 556
 557         do {
 558                 idx = find_next_zero_bit(map, end, start);
 559                 if (idx == end)
 560                         return -ENOSPC;
 561         } while (test_and_set_bit(idx, map));
 562
 563         return idx;
 564 }
 565
 566 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 567 {
 568         clear_bit(idx, map);
 569 }
 570
 571 /* Wait for any pending TLB invalidations to complete */
 572 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
 573 {
 574         int count = 0;
 575         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 576
 577         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
 578         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
 579                & sTLBGSTATUS_GSACTIVE) {
 580                 cpu_relax();
 581                 if (++count == TLB_LOOP_TIMEOUT) {
 582                         dev_err_ratelimited(smmu->dev,
 583                         "TLB sync timed out -- SMMU may be deadlocked\n");
 584                         return;
 585                 }
 586                 udelay(1);
 587         }
 588 }
 589
 590 static void arm_smmu_tlb_sync(void *cookie)
 591 {
 592         struct arm_smmu_domain *smmu_domain = cookie;
 593         __arm_smmu_tlb_sync(smmu_domain->smmu);
 594 }
 595
 596 static void arm_smmu_tlb_inv_context(void *cookie)
 597 {
 598         struct arm_smmu_domain *smmu_domain = cookie;
 599         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 600         struct arm_smmu_device *smmu = smmu_domain->smmu;
 601         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 602         void __iomem *base;
 603
 604         if (stage1) {
 605                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 606                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
 607                                base + ARM_SMMU_CB_S1_TLBIASID);
 608         } else {
 609                 base = ARM_SMMU_GR0(smmu);
 610                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
 611                                base + ARM_SMMU_GR0_TLBIVMID);
 612         }
 613
 614         __arm_smmu_tlb_sync(smmu);
 615 }
 616
 617 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 618                                           size_t granule, bool leaf, void *cookie)
 619 {
 620         struct arm_smmu_domain *smmu_domain = cookie;
 621         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 622         struct arm_smmu_device *smmu = smmu_domain->smmu;
 623         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 624         void __iomem *reg;
 625
 626         if (stage1) {
 627                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 628                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 629
 630                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 631                         iova &= ~12UL;
 632                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
 633                         do {
 634                                 writel_relaxed(iova, reg);
 635                                 iova += granule;
 636                         } while (size -= granule);
 637                 } else {
 638                         iova >>= 12;
 639                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
 640                         do {
 641                                 writeq_relaxed(iova, reg);
 642                                 iova += granule >> 12;
 643                         } while (size -= granule);
 644                 }
 645         } else if (smmu->version == ARM_SMMU_V2) {
 646                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 647                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 648                               ARM_SMMU_CB_S2_TLBIIPAS2;
 649                 iova >>= 12;
 650                 do {
 651                         smmu_write_atomic_lq(iova, reg);
 652                         iova += granule >> 12;
 653                 } while (size -= granule);
 654         } else {
 655                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
 656                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
 657         }
 658 }
 659
 660 static const struct iommu_gather_ops arm_smmu_gather_ops = {
 661         .tlb_flush_all  = arm_smmu_tlb_inv_context,
 662         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 663         .tlb_sync       = arm_smmu_tlb_sync,
 664 };
 665
 666 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 667 {
 668         u32 fsr, fsynr;
 669         unsigned long iova;
 670         struct iommu_domain *domain = dev;
 671         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 672         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 673         struct arm_smmu_device *smmu = smmu_domain->smmu;
 674         void __iomem *cb_base;
 675
 676         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 677         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 678
 679         if (!(fsr & FSR_FAULT))
 680                 return IRQ_NONE;
 681
 682         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 683         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 684
 685         dev_err_ratelimited(smmu->dev,
 686         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 687                             fsr, iova, fsynr, cfg->cbndx);
 688
 689         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 690         return IRQ_HANDLED;
 691 }
 692
 693 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 694 {
 695         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 696         struct arm_smmu_device *smmu = dev;
 697         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 698
 699         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 700         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 701         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 702         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 703
 704         if (!gfsr)
 705                 return IRQ_NONE;
 706
 707         dev_err_ratelimited(smmu->dev,
 708                 "Unexpected global fault, this could be serious\n");
 709         dev_err_ratelimited(smmu->dev,
 710                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 711                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 712
 713         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 714         return IRQ_HANDLED;
 715 }
 716
 717 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 718                                        struct io_pgtable_cfg *pgtbl_cfg)
 719 {
 720         u32 reg, reg2;
 721         u64 reg64;
 722         bool stage1;
 723         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 724         struct arm_smmu_device *smmu = smmu_domain->smmu;
 725         void __iomem *cb_base, *gr1_base;
 726
 727         gr1_base = ARM_SMMU_GR1(smmu);
 728         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 729         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 730
 731         if (smmu->version > ARM_SMMU_V1) {
 732                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 733                         reg = CBA2R_RW64_64BIT;
 734                 else
 735                         reg = CBA2R_RW64_32BIT;
 736                 /* 16-bit VMIDs live in CBA2R */
 737                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 738                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
 739
 740                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 741         }
 742
 743         /* CBAR */
 744         reg = cfg->cbar;
 745         if (smmu->version < ARM_SMMU_V2)
 746                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 747
 748         /*
 749          * Use the weakest shareability/memory types, so they are
 750          * overridden by the ttbcr/pte.
 751          */
 752         if (stage1) {
 753                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 754                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 755         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 756                 /* 8-bit VMIDs live in CBAR */
 757                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
 758         }
 759         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 760
 761         /* TTBRs */
 762         if (stage1) {
 763                 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
 764
 765                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 766                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 767                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
 768                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 769                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
 770                         writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 771                 } else {
 772                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 773                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 774                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 775                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 776                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 777                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 778                 }
 779         } else {
 780                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 781                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 782         }
 783
 784         /* TTBCR */
 785         if (stage1) {
 786                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 787                         reg = pgtbl_cfg->arm_v7s_cfg.tcr;
 788                         reg2 = 0;
 789                 } else {
 790                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 791                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 792                         reg2 |= TTBCR2_SEP_UPSTREAM;
 793                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 794                                 reg2 |= TTBCR2_AS;
 795                 }
 796                 if (smmu->version > ARM_SMMU_V1)
 797                         writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
 798         } else {
 799                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 800         }
 801         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 802
 803         /* MAIRs (stage-1 only) */
 804         if (stage1) {
 805                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 806                         reg = pgtbl_cfg->arm_v7s_cfg.prrr;
 807                         reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
 808                 } else {
 809                         reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 810                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 811                 }
 812                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 813                 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
 814         }
 815
 816         /* SCTLR */
 817         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 818         if (stage1)
 819                 reg |= SCTLR_S1_ASIDPNE;
 820 #ifdef __BIG_ENDIAN
 821         reg |= SCTLR_E;
 822 #endif
 823         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 824 }
 825
 826 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 827                                         struct arm_smmu_device *smmu)
 828 {
 829         int irq, start, ret = 0;
 830         unsigned long ias, oas;
 831         struct io_pgtable_ops *pgtbl_ops;
 832         struct io_pgtable_cfg pgtbl_cfg;
 833         enum io_pgtable_fmt fmt;
 834         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 835         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 836
 837         mutex_lock(&smmu_domain->init_mutex);
 838         if (smmu_domain->smmu)
 839                 goto out_unlock;
 840
 841         /*
 842          * Mapping the requested stage onto what we support is surprisingly
 843          * complicated, mainly because the spec allows S1+S2 SMMUs without
 844          * support for nested translation. That means we end up with the
 845          * following table:
 846          *
 847          * Requested        Supported        Actual
 848          *     S1               N              S1
 849          *     S1             S1+S2            S1
 850          *     S1               S2             S2
 851          *     S1               S1             S1
 852          *     N                N              N
 853          *     N              S1+S2            S2
 854          *     N                S2             S2
 855          *     N                S1             S1
 856          *
 857          * Note that you can't actually request stage-2 mappings.
 858          */
 859         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 860                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 861         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 862                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 863
 864         /*
 865          * Choosing a suitable context format is even more fiddly. Until we
 866          * grow some way for the caller to express a preference, and/or move
 867          * the decision into the io-pgtable code where it arguably belongs,
 868          * just aim for the closest thing to the rest of the system, and hope
 869          * that the hardware isn't esoteric enough that we can't assume AArch64
 870          * support to be a superset of AArch32 support...
 871          */
 872         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 873                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 874         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 875             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 876             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 877             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 878                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 879         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 880             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 881                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 882                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 883                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 884
 885         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 886                 ret = -EINVAL;
 887                 goto out_unlock;
 888         }
 889
 890         switch (smmu_domain->stage) {
 891         case ARM_SMMU_DOMAIN_S1:
 892                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 893                 start = smmu->num_s2_context_banks;
 894                 ias = smmu->va_size;
 895                 oas = smmu->ipa_size;
 896                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 897                         fmt = ARM_64_LPAE_S1;
 898                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 899                         fmt = ARM_32_LPAE_S1;
 900                         ias = min(ias, 32UL);
 901                         oas = min(oas, 40UL);
 902                 } else {
 903                         fmt = ARM_V7S;
 904                         ias = min(ias, 32UL);
 905                         oas = min(oas, 32UL);
 906                 }
 907                 break;
 908         case ARM_SMMU_DOMAIN_NESTED:
 909                 /*
 910                  * We will likely want to change this if/when KVM gets
 911                  * involved.
 912                  */
 913         case ARM_SMMU_DOMAIN_S2:
 914                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 915                 start = 0;
 916                 ias = smmu->ipa_size;
 917                 oas = smmu->pa_size;
 918                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 919                         fmt = ARM_64_LPAE_S2;
 920                 } else {
 921                         fmt = ARM_32_LPAE_S2;
 922                         ias = min(ias, 40UL);
 923                         oas = min(oas, 40UL);
 924                 }
 925                 break;
 926         default:
 927                 ret = -EINVAL;
 928                 goto out_unlock;
 929         }
 930
 931         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 932                                       smmu->num_context_banks);
 933         if (ret < 0)
 934                 goto out_unlock;
 935
 936         cfg->cbndx = ret;
 937         if (smmu->version < ARM_SMMU_V2) {
 938                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 939                 cfg->irptndx %= smmu->num_context_irqs;
 940         } else {
 941                 cfg->irptndx = cfg->cbndx;
 942         }
 943
 944         pgtbl_cfg = (struct io_pgtable_cfg) {
 945                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 946                 .ias            = ias,
 947                 .oas            = oas,
 948                 .tlb            = &arm_smmu_gather_ops,
 949                 .iommu_dev      = smmu->dev,
 950         };
 951
 952         smmu_domain->smmu = smmu;
 953         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 954         if (!pgtbl_ops) {
 955                 ret = -ENOMEM;
 956                 goto out_clear_smmu;
 957         }
 958
 959         /* Update the domain's page sizes to reflect the page table format */
 960         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 961         domain->geometry.aperture_end = (1UL << ias) - 1;
 962         domain->geometry.force_aperture = true;
 963
 964         /* Initialise the context bank with our page table cfg */
 965         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 966
 967         /*
 968          * Request context fault interrupt. Do this last to avoid the
 969          * handler seeing a half-initialised domain state.
 970          */
 971         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 972         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 973                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 974         if (ret < 0) {
 975                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 976                         cfg->irptndx, irq);
 977                 cfg->irptndx = INVALID_IRPTNDX;
 978         }
 979
 980         mutex_unlock(&smmu_domain->init_mutex);
 981
 982         /* Publish page table ops for map/unmap */
 983         smmu_domain->pgtbl_ops = pgtbl_ops;
 984         return 0;
 985
 986 out_clear_smmu:
 987         smmu_domain->smmu = NULL;
 988 out_unlock:
 989         mutex_unlock(&smmu_domain->init_mutex);
 990         return ret;
 991 }
 992
 993 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 994 {
 995         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 996         struct arm_smmu_device *smmu = smmu_domain->smmu;
 997         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 998         void __iomem *cb_base;
 999         int irq;
1000
1001         if (!smmu)
1002                 return;
1003
1004         /*
1005          * Disable the context bank and free the page tables before freeing
1006          * it.
1007          */
1008         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1009         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1010
1011         if (cfg->irptndx != INVALID_IRPTNDX) {
1012                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1013                 devm_free_irq(smmu->dev, irq, domain);
1014         }
1015
1016         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1017         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1018 }
1019
1020 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1021 {
1022         struct arm_smmu_domain *smmu_domain;
1023
1024         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1025                 return NULL;
1026         /*
1027          * Allocate the domain and initialise some of its data structures.
1028          * We can't really do anything meaningful until we've added a
1029          * master.
1030          */
1031         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1032         if (!smmu_domain)
1033                 return NULL;
1034
1035         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1036             iommu_get_dma_cookie(&smmu_domain->domain))) {
1037                 kfree(smmu_domain);
1038                 return NULL;
1039         }
1040
1041         mutex_init(&smmu_domain->init_mutex);
1042         spin_lock_init(&smmu_domain->pgtbl_lock);
1043
1044         return &smmu_domain->domain;
1045 }
1046
1047 static void arm_smmu_domain_free(struct iommu_domain *domain)
1048 {
1049         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1050
1051         /*
1052          * Free the domain resources. We assume that all devices have
1053          * already been detached.
1054          */
1055         iommu_put_dma_cookie(domain);
1056         arm_smmu_destroy_domain_context(domain);
1057         kfree(smmu_domain);
1058 }
1059
1060 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1061 {
1062         struct arm_smmu_smr *smr = smmu->smrs + idx;
1063         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1064
1065         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
1066                 reg |= SMR_VALID;
1067         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1068 }
1069
1070 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1071 {
1072         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1073         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1074                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1075                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1076
1077         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
1078             smmu->smrs[idx].valid)
1079                 reg |= S2CR_EXIDVALID;
1080         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1081 }
1082
1083 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1084 {
1085         arm_smmu_write_s2cr(smmu, idx);
1086         if (smmu->smrs)
1087                 arm_smmu_write_smr(smmu, idx);
1088 }
1089
1090 /*
1091  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
1092  * should be called after sCR0 is written.
1093  */
1094 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
1095 {
1096         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1097         u32 smr;
1098
1099         if (!smmu->smrs)
1100                 return;
1101
1102         /*
1103          * SMR.ID bits may not be preserved if the corresponding MASK
1104          * bits are set, so check each one separately. We can reject
1105          * masters later if they try to claim IDs outside these masks.
1106          */
1107         smr = smmu->streamid_mask << SMR_ID_SHIFT;
1108         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1109         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1110         smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1111
1112         smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1113         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1114         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1115         smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1116 }
1117
1118 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1119 {
1120         struct arm_smmu_smr *smrs = smmu->smrs;
1121         int i, free_idx = -ENOSPC;
1122
1123         /* Stream indexing is blissfully easy */
1124         if (!smrs)
1125                 return id;
1126
1127         /* Validating SMRs is... less so */
1128         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1129                 if (!smrs[i].valid) {
1130                         /*
1131                          * Note the first free entry we come across, which
1132                          * we'll claim in the end if nothing else matches.
1133                          */
1134                         if (free_idx < 0)
1135                                 free_idx = i;
1136                         continue;
1137                 }
1138                 /*
1139                  * If the new entry is _entirely_ matched by an existing entry,
1140                  * then reuse that, with the guarantee that there also cannot
1141                  * be any subsequent conflicting entries. In normal use we'd
1142                  * expect simply identical entries for this case, but there's
1143                  * no harm in accommodating the generalisation.
1144                  */
1145                 if ((mask & smrs[i].mask) == mask &&
1146                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1147                         return i;
1148                 /*
1149                  * If the new entry has any other overlap with an existing one,
1150                  * though, then there always exists at least one stream ID
1151                  * which would cause a conflict, and we can't allow that risk.
1152                  */
1153                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1154                         return -EINVAL;
1155         }
1156
1157         return free_idx;
1158 }
1159
1160 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1161 {
1162         if (--smmu->s2crs[idx].count)
1163                 return false;
1164
1165         smmu->s2crs[idx] = s2cr_init_val;
1166         if (smmu->smrs)
1167                 smmu->smrs[idx].valid = false;
1168
1169         return true;
1170 }
1171
1172 static int arm_smmu_master_alloc_smes(struct device *dev)
1173 {
1174         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1175         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1176         struct arm_smmu_device *smmu = cfg->smmu;
1177         struct arm_smmu_smr *smrs = smmu->smrs;
1178         struct iommu_group *group;
1179         int i, idx, ret;
1180
1181         mutex_lock(&smmu->stream_map_mutex);
1182         /* Figure out a viable stream map entry allocation */
1183         for_each_cfg_sme(fwspec, i, idx) {
1184                 u16 sid = fwspec->ids[i];
1185                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1186
1187                 if (idx != INVALID_SMENDX) {
1188                         ret = -EEXIST;
1189                         goto out_err;
1190                 }
1191
1192                 ret = arm_smmu_find_sme(smmu, sid, mask);
1193                 if (ret < 0)
1194                         goto out_err;
1195
1196                 idx = ret;
1197                 if (smrs && smmu->s2crs[idx].count == 0) {
1198                         smrs[idx].id = sid;
1199                         smrs[idx].mask = mask;
1200                         smrs[idx].valid = true;
1201                 }
1202                 smmu->s2crs[idx].count++;
1203                 cfg->smendx[i] = (s16)idx;
1204         }
1205
1206         group = iommu_group_get_for_dev(dev);
1207         if (!group)
1208                 group = ERR_PTR(-ENOMEM);
1209         if (IS_ERR(group)) {
1210                 ret = PTR_ERR(group);
1211                 goto out_err;
1212         }
1213         iommu_group_put(group);
1214
1215         /* It worked! Now, poke the actual hardware */
1216         for_each_cfg_sme(fwspec, i, idx) {
1217                 arm_smmu_write_sme(smmu, idx);
1218                 smmu->s2crs[idx].group = group;
1219         }
1220
1221         mutex_unlock(&smmu->stream_map_mutex);
1222         return 0;
1223
1224 out_err:
1225         while (i--) {
1226                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1227                 cfg->smendx[i] = INVALID_SMENDX;
1228         }
1229         mutex_unlock(&smmu->stream_map_mutex);
1230         return ret;
1231 }
1232
1233 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1234 {
1235         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1236         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1237         int i, idx;
1238
1239         mutex_lock(&smmu->stream_map_mutex);
1240         for_each_cfg_sme(fwspec, i, idx) {
1241                 if (arm_smmu_free_sme(smmu, idx))
1242                         arm_smmu_write_sme(smmu, idx);
1243                 cfg->smendx[i] = INVALID_SMENDX;
1244         }
1245         mutex_unlock(&smmu->stream_map_mutex);
1246 }
1247
1248 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1249                                       struct iommu_fwspec *fwspec)
1250 {
1251         struct arm_smmu_device *smmu = smmu_domain->smmu;
1252         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1253         enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1254         u8 cbndx = smmu_domain->cfg.cbndx;
1255         int i, idx;
1256
1257         for_each_cfg_sme(fwspec, i, idx) {
1258                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1259                         continue;
1260
1261                 s2cr[idx].type = type;
1262                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1263                 s2cr[idx].cbndx = cbndx;
1264                 arm_smmu_write_s2cr(smmu, idx);
1265         }
1266         return 0;
1267 }
1268
1269 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1270 {
1271         int ret;
1272         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1273         struct arm_smmu_device *smmu;
1274         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1275
1276         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1277                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1278                 return -ENXIO;
1279         }
1280
1281         /*
1282          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1283          * domains between of_xlate() and add_device() - we have no way to cope
1284          * with that, so until ARM gets converted to rely on groups and default
1285          * domains, just say no (but more politely than by dereferencing NULL).
1286          * This should be at least a WARN_ON once that's sorted.
1287          */
1288         if (!fwspec->iommu_priv)
1289                 return -ENODEV;
1290
1291         smmu = fwspec_smmu(fwspec);
1292         /* Ensure that the domain is finalised */
1293         ret = arm_smmu_init_domain_context(domain, smmu);
1294         if (ret < 0)
1295                 return ret;
1296
1297         /*
1298          * Sanity check the domain. We don't support domains across
1299          * different SMMUs.
1300          */
1301         if (smmu_domain->smmu != smmu) {
1302                 dev_err(dev,
1303                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1304                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1305                 return -EINVAL;
1306         }
1307
1308         /* Looks ok, so add the device to the domain */
1309         return arm_smmu_domain_add_master(smmu_domain, fwspec);
1310 }
1311
1312 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1313                         phys_addr_t paddr, size_t size, int prot)
1314 {
1315         int ret;
1316         unsigned long flags;
1317         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1318         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1319
1320         if (!ops)
1321                 return -ENODEV;
1322
1323         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1324         ret = ops->map(ops, iova, paddr, size, prot);
1325         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1326         return ret;
1327 }
1328
1329 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1330                              size_t size)
1331 {
1332         size_t ret;
1333         unsigned long flags;
1334         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1335         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1336
1337         if (!ops)
1338                 return 0;
1339
1340         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1341         ret = ops->unmap(ops, iova, size);
1342         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1343         return ret;
1344 }
1345
1346 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1347                                               dma_addr_t iova)
1348 {
1349         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1350         struct arm_smmu_device *smmu = smmu_domain->smmu;
1351         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1352         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1353         struct device *dev = smmu->dev;
1354         void __iomem *cb_base;
1355         u32 tmp;
1356         u64 phys;
1357         unsigned long va;
1358
1359         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1360
1361         /* ATS1 registers can only be written atomically */
1362         va = iova & ~0xfffUL;
1363         if (smmu->version == ARM_SMMU_V2)
1364                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1365         else /* Register is only 32-bit in v1 */
1366                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1367
1368         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1369                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1370                 dev_err(dev,
1371                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1372                         &iova);
1373                 return ops->iova_to_phys(ops, iova);
1374         }
1375
1376         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1377         if (phys & CB_PAR_F) {
1378                 dev_err(dev, "translation fault!\n");
1379                 dev_err(dev, "PAR = 0x%llx\n", phys);
1380                 return 0;
1381         }
1382
1383         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1384 }
1385
1386 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1387                                         dma_addr_t iova)
1388 {
1389         phys_addr_t ret;
1390         unsigned long flags;
1391         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1392         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1393
1394         if (!ops)
1395                 return 0;
1396
1397         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1398         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1399                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1400                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1401         } else {
1402                 ret = ops->iova_to_phys(ops, iova);
1403         }
1404
1405         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1406
1407         return ret;
1408 }
1409
1410 static bool arm_smmu_capable(enum iommu_cap cap)
1411 {
1412         switch (cap) {
1413         case IOMMU_CAP_CACHE_COHERENCY:
1414                 /*
1415                  * Return true here as the SMMU can always send out coherent
1416                  * requests.
1417                  */
1418                 return true;
1419         case IOMMU_CAP_NOEXEC:
1420                 return true;
1421         default:
1422                 return false;
1423         }
1424 }
1425
1426 static int arm_smmu_match_node(struct device *dev, void *data)
1427 {
1428         return dev->fwnode == data;
1429 }
1430
1431 static
1432 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1433 {
1434         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1435                                                 fwnode, arm_smmu_match_node);
1436         put_device(dev);
1437         return dev ? dev_get_drvdata(dev) : NULL;
1438 }
1439
1440 static int arm_smmu_add_device(struct device *dev)
1441 {
1442         struct arm_smmu_device *smmu;
1443         struct arm_smmu_master_cfg *cfg;
1444         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1445         int i, ret;
1446
1447         if (using_legacy_binding) {
1448                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1449                 fwspec = dev->iommu_fwspec;
1450                 if (ret)
1451                         goto out_free;
1452         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1453                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1454         } else {
1455                 return -ENODEV;
1456         }
1457
1458         ret = -EINVAL;
1459         for (i = 0; i < fwspec->num_ids; i++) {
1460                 u16 sid = fwspec->ids[i];
1461                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1462
1463                 if (sid & ~smmu->streamid_mask) {
1464                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1465                                 sid, smmu->streamid_mask);
1466                         goto out_free;
1467                 }
1468                 if (mask & ~smmu->smr_mask_mask) {
1469                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1470                                 sid, smmu->smr_mask_mask);
1471                         goto out_free;
1472                 }
1473         }
1474
1475         ret = -ENOMEM;
1476         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1477                       GFP_KERNEL);
1478         if (!cfg)
1479                 goto out_free;
1480
1481         cfg->smmu = smmu;
1482         fwspec->iommu_priv = cfg;
1483         while (i--)
1484                 cfg->smendx[i] = INVALID_SMENDX;
1485
1486         ret = arm_smmu_master_alloc_smes(dev);
1487         if (ret)
1488                 goto out_free;
1489
1490         iommu_device_link(&smmu->iommu, dev);
1491
1492         return 0;
1493
1494 out_free:
1495         if (fwspec)
1496                 kfree(fwspec->iommu_priv);
1497         iommu_fwspec_free(dev);
1498         return ret;
1499 }
1500
1501 static void arm_smmu_remove_device(struct device *dev)
1502 {
1503         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1504         struct arm_smmu_master_cfg *cfg;
1505         struct arm_smmu_device *smmu;
1506
1507
1508         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1509                 return;
1510
1511         cfg  = fwspec->iommu_priv;
1512         smmu = cfg->smmu;
1513
1514         iommu_device_unlink(&smmu->iommu, dev);
1515         arm_smmu_master_free_smes(fwspec);
1516         iommu_group_remove_device(dev);
1517         kfree(fwspec->iommu_priv);
1518         iommu_fwspec_free(dev);
1519 }
1520
1521 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1522 {
1523         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1524         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1525         struct iommu_group *group = NULL;
1526         int i, idx;
1527
1528         for_each_cfg_sme(fwspec, i, idx) {
1529                 if (group && smmu->s2crs[idx].group &&
1530                     group != smmu->s2crs[idx].group)
1531                         return ERR_PTR(-EINVAL);
1532
1533                 group = smmu->s2crs[idx].group;
1534         }
1535
1536         if (group)
1537                 return iommu_group_ref_get(group);
1538
1539         if (dev_is_pci(dev))
1540                 group = pci_device_group(dev);
1541         else
1542                 group = generic_device_group(dev);
1543
1544         return group;
1545 }
1546
1547 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1548                                     enum iommu_attr attr, void *data)
1549 {
1550         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1551
1552         switch (attr) {
1553         case DOMAIN_ATTR_NESTING:
1554                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1555                 return 0;
1556         default:
1557                 return -ENODEV;
1558         }
1559 }
1560
1561 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1562                                     enum iommu_attr attr, void *data)
1563 {
1564         int ret = 0;
1565         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1566
1567         mutex_lock(&smmu_domain->init_mutex);
1568
1569         switch (attr) {
1570         case DOMAIN_ATTR_NESTING:
1571                 if (smmu_domain->smmu) {
1572                         ret = -EPERM;
1573                         goto out_unlock;
1574                 }
1575
1576                 if (*(int *)data)
1577                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1578                 else
1579                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1580
1581                 break;
1582         default:
1583                 ret = -ENODEV;
1584         }
1585
1586 out_unlock:
1587         mutex_unlock(&smmu_domain->init_mutex);
1588         return ret;
1589 }
1590
1591 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1592 {
1593         u32 fwid = 0;
1594
1595         if (args->args_count > 0)
1596                 fwid |= (u16)args->args[0];
1597
1598         if (args->args_count > 1)
1599                 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1600
1601         return iommu_fwspec_add_ids(dev, &fwid, 1);
1602 }
1603
1604 static void arm_smmu_get_resv_regions(struct device *dev,
1605                                       struct list_head *head)
1606 {
1607         struct iommu_resv_region *region;
1608         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1609
1610         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1611                                          prot, IOMMU_RESV_SW_MSI);
1612         if (!region)
1613                 return;
1614
1615         list_add_tail(&region->list, head);
1616
1617         iommu_dma_get_resv_regions(dev, head);
1618 }
1619
1620 static void arm_smmu_put_resv_regions(struct device *dev,
1621                                       struct list_head *head)
1622 {
1623         struct iommu_resv_region *entry, *next;
1624
1625         list_for_each_entry_safe(entry, next, head, list)
1626                 kfree(entry);
1627 }
1628
1629 static struct iommu_ops arm_smmu_ops = {
1630         .capable                = arm_smmu_capable,
1631         .domain_alloc           = arm_smmu_domain_alloc,
1632         .domain_free            = arm_smmu_domain_free,
1633         .attach_dev             = arm_smmu_attach_dev,
1634         .map                    = arm_smmu_map,
1635         .unmap                  = arm_smmu_unmap,
1636         .map_sg                 = default_iommu_map_sg,
1637         .iova_to_phys           = arm_smmu_iova_to_phys,
1638         .add_device             = arm_smmu_add_device,
1639         .remove_device          = arm_smmu_remove_device,
1640         .device_group           = arm_smmu_device_group,
1641         .domain_get_attr        = arm_smmu_domain_get_attr,
1642         .domain_set_attr        = arm_smmu_domain_set_attr,
1643         .of_xlate               = arm_smmu_of_xlate,
1644         .get_resv_regions       = arm_smmu_get_resv_regions,
1645         .put_resv_regions       = arm_smmu_put_resv_regions,
1646         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1647 };
1648
1649 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1650 {
1651         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1652         void __iomem *cb_base;
1653         int i;
1654         u32 reg, major;
1655
1656         /* clear global FSR */
1657         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1658         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1659
1660         /*
1661          * Reset stream mapping groups: Initial values mark all SMRn as
1662          * invalid and all S2CRn as bypass unless overridden.
1663          */
1664         for (i = 0; i < smmu->num_mapping_groups; ++i)
1665                 arm_smmu_write_sme(smmu, i);
1666
1667         if (smmu->model == ARM_MMU500) {
1668                 /*
1669                  * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1670                  * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1671                  * bit is only present in MMU-500r2 onwards.
1672                  */
1673                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1674                 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1675                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1676                 if (major >= 2)
1677                         reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1678                 /*
1679                  * Allow unmatched Stream IDs to allocate bypass
1680                  * TLB entries for reduced latency.
1681                  */
1682                 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN;
1683                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1684         }
1685
1686         /* Make sure all context banks are disabled and clear CB_FSR  */
1687         for (i = 0; i < smmu->num_context_banks; ++i) {
1688                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1689                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1690                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1691                 /*
1692                  * Disable MMU-500's not-particularly-beneficial next-page
1693                  * prefetcher for the sake of errata #841119 and #826419.
1694                  */
1695                 if (smmu->model == ARM_MMU500) {
1696                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1697                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1698                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1699                 }
1700         }
1701
1702         /* Invalidate the TLB, just in case */
1703         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1704         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1705
1706         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1707
1708         /* Enable fault reporting */
1709         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1710
1711         /* Disable TLB broadcasting. */
1712         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1713
1714         /* Enable client access, handling unmatched streams as appropriate */
1715         reg &= ~sCR0_CLIENTPD;
1716         if (disable_bypass)
1717                 reg |= sCR0_USFCFG;
1718         else
1719                 reg &= ~sCR0_USFCFG;
1720
1721         /* Disable forced broadcasting */
1722         reg &= ~sCR0_FB;
1723
1724         /* Don't upgrade barriers */
1725         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1726
1727         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1728                 reg |= sCR0_VMID16EN;
1729
1730         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1731                 reg |= sCR0_EXIDENABLE;
1732
1733         /* Push the button */
1734         __arm_smmu_tlb_sync(smmu);
1735         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1736 }
1737
1738 static int arm_smmu_id_size_to_bits(int size)
1739 {
1740         switch (size) {
1741         case 0:
1742                 return 32;
1743         case 1:
1744                 return 36;
1745         case 2:
1746                 return 40;
1747         case 3:
1748                 return 42;
1749         case 4:
1750                 return 44;
1751         case 5:
1752         default:
1753                 return 48;
1754         }
1755 }
1756
1757 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1758 {
1759         unsigned long size;
1760         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1761         u32 id;
1762         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1763         int i;
1764
1765         dev_notice(smmu->dev, "probing hardware configuration...\n");
1766         dev_notice(smmu->dev, "SMMUv%d with:\n",
1767                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1768
1769         /* ID0 */
1770         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1771
1772         /* Restrict available stages based on module parameter */
1773         if (force_stage == 1)
1774                 id &= ~(ID0_S2TS | ID0_NTS);
1775         else if (force_stage == 2)
1776                 id &= ~(ID0_S1TS | ID0_NTS);
1777
1778         if (id & ID0_S1TS) {
1779                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1780                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1781         }
1782
1783         if (id & ID0_S2TS) {
1784                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1785                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1786         }
1787
1788         if (id & ID0_NTS) {
1789                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1790                 dev_notice(smmu->dev, "\tnested translation\n");
1791         }
1792
1793         if (!(smmu->features &
1794                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1795                 dev_err(smmu->dev, "\tno translation support!\n");
1796                 return -ENODEV;
1797         }
1798
1799         if ((id & ID0_S1TS) &&
1800                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1801                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1802                 dev_notice(smmu->dev, "\taddress translation ops\n");
1803         }
1804
1805         /*
1806          * In order for DMA API calls to work properly, we must defer to what
1807          * the FW says about coherency, regardless of what the hardware claims.
1808          * Fortunately, this also opens up a workaround for systems where the
1809          * ID register value has ended up configured incorrectly.
1810          */
1811         cttw_reg = !!(id & ID0_CTTW);
1812         if (cttw_fw || cttw_reg)
1813                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1814                            cttw_fw ? "" : "non-");
1815         if (cttw_fw != cttw_reg)
1816                 dev_notice(smmu->dev,
1817                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1818
1819         /* Max. number of entries we have for stream matching/indexing */
1820         if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1821                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1822                 size = 1 << 16;
1823         } else {
1824                 size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1825         }
1826         smmu->streamid_mask = size - 1;
1827         if (id & ID0_SMS) {
1828                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1829                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1830                 if (size == 0) {
1831                         dev_err(smmu->dev,
1832                                 "stream-matching supported, but no SMRs present!\n");
1833                         return -ENODEV;
1834                 }
1835
1836                 /* Zero-initialised to mark as invalid */
1837                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1838                                           GFP_KERNEL);
1839                 if (!smmu->smrs)
1840                         return -ENOMEM;
1841
1842                 dev_notice(smmu->dev,
1843                            "\tstream matching with %lu register groups", size);
1844         }
1845         /* s2cr->type == 0 means translation, so initialise explicitly */
1846         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1847                                          GFP_KERNEL);
1848         if (!smmu->s2crs)
1849                 return -ENOMEM;
1850         for (i = 0; i < size; i++)
1851                 smmu->s2crs[i] = s2cr_init_val;
1852
1853         smmu->num_mapping_groups = size;
1854         mutex_init(&smmu->stream_map_mutex);
1855
1856         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1857                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1858                 if (!(id & ID0_PTFS_NO_AARCH32S))
1859                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1860         }
1861
1862         /* ID1 */
1863         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1864         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1865
1866         /* Check for size mismatch of SMMU address space from mapped region */
1867         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1868         size *= 2 << smmu->pgshift;
1869         if (smmu->size != size)
1870                 dev_warn(smmu->dev,
1871                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1872                         size, smmu->size);
1873
1874         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1875         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1876         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1877                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1878                 return -ENODEV;
1879         }
1880         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1881                    smmu->num_context_banks, smmu->num_s2_context_banks);
1882         /*
1883          * Cavium CN88xx erratum #27704.
1884          * Ensure ASID and VMID allocation is unique across all SMMUs in
1885          * the system.
1886          */
1887         if (smmu->model == CAVIUM_SMMUV2) {
1888                 smmu->cavium_id_base =
1889                         atomic_add_return(smmu->num_context_banks,
1890                                           &cavium_smmu_context_count);
1891                 smmu->cavium_id_base -= smmu->num_context_banks;
1892         }
1893
1894         /* ID2 */
1895         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1896         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1897         smmu->ipa_size = size;
1898
1899         /* The output mask is also applied for bypass */
1900         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1901         smmu->pa_size = size;
1902
1903         if (id & ID2_VMID16)
1904                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1905
1906         /*
1907          * What the page table walker can address actually depends on which
1908          * descriptor format is in use, but since a) we don't know that yet,
1909          * and b) it can vary per context bank, this will have to do...
1910          */
1911         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1912                 dev_warn(smmu->dev,
1913                          "failed to set DMA mask for table walker\n");
1914
1915         if (smmu->version < ARM_SMMU_V2) {
1916                 smmu->va_size = smmu->ipa_size;
1917                 if (smmu->version == ARM_SMMU_V1_64K)
1918                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1919         } else {
1920                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1921                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1922                 if (id & ID2_PTFS_4K)
1923                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1924                 if (id & ID2_PTFS_16K)
1925                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1926                 if (id & ID2_PTFS_64K)
1927                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1928         }
1929
1930         /* Now we've corralled the various formats, what'll it do? */
1931         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1932                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1933         if (smmu->features &
1934             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1935                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1936         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1937                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1938         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1939                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1940
1941         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1942                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1943         else
1944                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1945         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1946                    smmu->pgsize_bitmap);
1947
1948
1949         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1950                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1951                            smmu->va_size, smmu->ipa_size);
1952
1953         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1954                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1955                            smmu->ipa_size, smmu->pa_size);
1956
1957         return 0;
1958 }
1959
1960 struct arm_smmu_match_data {
1961         enum arm_smmu_arch_version version;
1962         enum arm_smmu_implementation model;
1963 };
1964
1965 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1966 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1967
1968 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1969 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1970 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1971 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1972 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1973
1974 static const struct of_device_id arm_smmu_of_match[] = {
1975         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1976         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1977         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1978         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1979         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1980         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1981         { },
1982 };
1983 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1984
1985 #ifdef CONFIG_ACPI
1986 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1987 {
1988         int ret = 0;
1989
1990         switch (model) {
1991         case ACPI_IORT_SMMU_V1:
1992         case ACPI_IORT_SMMU_CORELINK_MMU400:
1993                 smmu->version = ARM_SMMU_V1;
1994                 smmu->model = GENERIC_SMMU;
1995                 break;
1996         case ACPI_IORT_SMMU_V2:
1997                 smmu->version = ARM_SMMU_V2;
1998                 smmu->model = GENERIC_SMMU;
1999                 break;
2000         case ACPI_IORT_SMMU_CORELINK_MMU500:
2001                 smmu->version = ARM_SMMU_V2;
2002                 smmu->model = ARM_MMU500;
2003                 break;
2004         default:
2005                 ret = -ENODEV;
2006         }
2007
2008         return ret;
2009 }
2010
2011 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2012                                       struct arm_smmu_device *smmu)
2013 {
2014         struct device *dev = smmu->dev;
2015         struct acpi_iort_node *node =
2016                 *(struct acpi_iort_node **)dev_get_platdata(dev);
2017         struct acpi_iort_smmu *iort_smmu;
2018         int ret;
2019
2020         /* Retrieve SMMU1/2 specific data */
2021         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2022
2023         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2024         if (ret < 0)
2025                 return ret;
2026
2027         /* Ignore the configuration access interrupt */
2028         smmu->num_global_irqs = 1;
2029
2030         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2031                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2032
2033         return 0;
2034 }
2035 #else
2036 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2037                                              struct arm_smmu_device *smmu)
2038 {
2039         return -ENODEV;
2040 }
2041 #endif
2042
2043 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2044                                     struct arm_smmu_device *smmu)
2045 {
2046         const struct arm_smmu_match_data *data;
2047         struct device *dev = &pdev->dev;
2048         bool legacy_binding;
2049
2050         if (of_property_read_u32(dev->of_node, "#global-interrupts",
2051                                  &smmu->num_global_irqs)) {
2052                 dev_err(dev, "missing #global-interrupts property\n");
2053                 return -ENODEV;
2054         }
2055
2056         data = of_device_get_match_data(dev);
2057         smmu->version = data->version;
2058         smmu->model = data->model;
2059
2060         parse_driver_options(smmu);
2061
2062         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2063         if (legacy_binding && !using_generic_binding) {
2064                 if (!using_legacy_binding)
2065                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2066                 using_legacy_binding = true;
2067         } else if (!legacy_binding && !using_legacy_binding) {
2068                 using_generic_binding = true;
2069         } else {
2070                 dev_err(dev, "not probing due to mismatched DT properties\n");
2071                 return -ENODEV;
2072         }
2073
2074         if (of_dma_is_coherent(dev->of_node))
2075                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2076
2077         return 0;
2078 }
2079
2080 static void arm_smmu_bus_init(void)
2081 {
2082         /* Oh, for a proper bus abstraction */
2083         if (!iommu_present(&platform_bus_type))
2084                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2085 #ifdef CONFIG_ARM_AMBA
2086         if (!iommu_present(&amba_bustype))
2087                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2088 #endif
2089 #ifdef CONFIG_PCI
2090         if (!iommu_present(&pci_bus_type)) {
2091                 pci_request_acs();
2092                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2093         }
2094 #endif
2095 }
2096
2097 static int arm_smmu_device_probe(struct platform_device *pdev)
2098 {
2099         struct resource *res;
2100         resource_size_t ioaddr;
2101         struct arm_smmu_device *smmu;
2102         struct device *dev = &pdev->dev;
2103         int num_irqs, i, err;
2104
2105         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2106         if (!smmu) {
2107                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2108                 return -ENOMEM;
2109         }
2110         smmu->dev = dev;
2111
2112         if (dev->of_node)
2113                 err = arm_smmu_device_dt_probe(pdev, smmu);
2114         else
2115                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2116
2117         if (err)
2118                 return err;
2119
2120         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2121         ioaddr = res->start;
2122         smmu->base = devm_ioremap_resource(dev, res);
2123         if (IS_ERR(smmu->base))
2124                 return PTR_ERR(smmu->base);
2125         smmu->size = resource_size(res);
2126
2127         num_irqs = 0;
2128         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2129                 num_irqs++;
2130                 if (num_irqs > smmu->num_global_irqs)
2131                         smmu->num_context_irqs++;
2132         }
2133
2134         if (!smmu->num_context_irqs) {
2135                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2136                         num_irqs, smmu->num_global_irqs + 1);
2137                 return -ENODEV;
2138         }
2139
2140         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
2141                                   GFP_KERNEL);
2142         if (!smmu->irqs) {
2143                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2144                 return -ENOMEM;
2145         }
2146
2147         for (i = 0; i < num_irqs; ++i) {
2148                 int irq = platform_get_irq(pdev, i);
2149
2150                 if (irq < 0) {
2151                         dev_err(dev, "failed to get irq index %d\n", i);
2152                         return -ENODEV;
2153                 }
2154                 smmu->irqs[i] = irq;
2155         }
2156
2157         err = arm_smmu_device_cfg_probe(smmu);
2158         if (err)
2159                 return err;
2160
2161         if (smmu->version == ARM_SMMU_V2 &&
2162             smmu->num_context_banks != smmu->num_context_irqs) {
2163                 dev_err(dev,
2164                         "found only %d context interrupt(s) but %d required\n",
2165                         smmu->num_context_irqs, smmu->num_context_banks);
2166                 return -ENODEV;
2167         }
2168
2169         for (i = 0; i < smmu->num_global_irqs; ++i) {
2170                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2171                                        arm_smmu_global_fault,
2172                                        IRQF_SHARED,
2173                                        "arm-smmu global fault",
2174                                        smmu);
2175                 if (err) {
2176                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2177                                 i, smmu->irqs[i]);
2178                         return err;
2179                 }
2180         }
2181
2182         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2183                                      "smmu.%pa", &ioaddr);
2184         if (err) {
2185                 dev_err(dev, "Failed to register iommu in sysfs\n");
2186                 return err;
2187         }
2188
2189         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2190         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2191
2192         err = iommu_device_register(&smmu->iommu);
2193         if (err) {
2194                 dev_err(dev, "Failed to register iommu\n");
2195                 return err;
2196         }
2197
2198         platform_set_drvdata(pdev, smmu);
2199         arm_smmu_device_reset(smmu);
2200         arm_smmu_test_smr_masks(smmu);
2201
2202         /*
2203          * For ACPI and generic DT bindings, an SMMU will be probed before
2204          * any device which might need it, so we want the bus ops in place
2205          * ready to handle default domain setup as soon as any SMMU exists.
2206          */
2207         if (!using_legacy_binding)
2208                 arm_smmu_bus_init();
2209
2210         return 0;
2211 }
2212
2213 /*
2214  * With the legacy DT binding in play, though, we have no guarantees about
2215  * probe order, but then we're also not doing default domains, so we can
2216  * delay setting bus ops until we're sure every possible SMMU is ready,
2217  * and that way ensure that no add_device() calls get missed.
2218  */
2219 static int arm_smmu_legacy_bus_init(void)
2220 {
2221         if (using_legacy_binding)
2222                 arm_smmu_bus_init();
2223         return 0;
2224 }
2225 device_initcall_sync(arm_smmu_legacy_bus_init);
2226
2227 static int arm_smmu_device_remove(struct platform_device *pdev)
2228 {
2229         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2230
2231         if (!smmu)
2232                 return -ENODEV;
2233
2234         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2235                 dev_err(&pdev->dev, "removing device with active domains!\n");
2236
2237         /* Turn the thing off */
2238         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2239         return 0;
2240 }
2241
2242 static struct platform_driver arm_smmu_driver = {
2243         .driver = {
2244                 .name           = "arm-smmu",
2245                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2246         },
2247         .probe  = arm_smmu_device_probe,
2248         .remove = arm_smmu_device_remove,
2249 };
2250 module_platform_driver(arm_smmu_driver);
2251
2252 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", NULL);
2253 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", NULL);
2254 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", NULL);
2255 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", NULL);
2256 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", NULL);
2257 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", NULL);
2258
2259 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2260 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2261 MODULE_LICENSE("GPL v2");